rocr: Generalize driver discovery

Generalize the driver discovery and move driver-specific
functionality to the concrete driver implementations.
Currently, this process is tightly coupled to the hsakmt
which is GPU and OS specific.

Change-Id: Ie1c53fef407a71b5ec4c6eaf3a3ed00871184409

libhsakmt/include/hsakmt/hsakmttypes.h

@@ -267,6 +267,8 @@ typedef struct _HsaNodeProperties
                                        // e.g a "discrete HSA GPU"
     HSAuint32       NumFComputeCores;  // # of HSA throughtput (= GPU) FCompute cores ("SIMD") present in a node.
                                        // This value is 0 if no FCompute cores are present (e.g. pure "CPU node").
+    HSAuint32 NumNeuralCores;          // # of HSA neural processing units (= AIE) present in a
+                                       // node. This value is 0 if there are no NeuralCores.
     HSAuint32       NumMemoryBanks;    // # of discoverable memory bank affinity properties on this "H-NUMA" node.
     HSAuint32       NumCaches;         // # of discoverable cache affinity properties on this "H-NUMA"  node.
 

runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp

@@ -42,34 +42,74 @@
 
 #include "core/inc/amd_kfd_driver.h"
 
-#include <sys/ioctl.h>
-
 #include <memory>
 #include <string>
 
+#include <link.h>
+#include <sys/ioctl.h>
+
 #include "hsakmt/hsakmt.h"
 
-#include "core/inc/amd_cpu_agent.h"
-#include "core/inc/amd_gpu_agent.h"
 #include "core/inc/amd_memory_region.h"
-#include "core/inc/exceptions.h"
 #include "core/inc/runtime.h"
 
+extern r_debug _amdgpu_r_debug;
+
 namespace rocr {
 namespace AMD {
 
 KfdDriver::KfdDriver(std::string devnode_name)
     : core::Driver(core::DriverType::KFD, devnode_name) {}
 
-hsa_status_t KfdDriver::Init() { return HSA_STATUS_SUCCESS; }
+hsa_status_t KfdDriver::Init() {
+  HSAKMT_STATUS ret =
+      hsaKmtRuntimeEnable(&_amdgpu_r_debug, core::Runtime::runtime_singleton_->flag().debug());
+
+  if (ret != HSAKMT_STATUS_SUCCESS && ret != HSAKMT_STATUS_NOT_SUPPORTED) return HSA_STATUS_ERROR;
+
+  uint32_t caps_mask = 0;
+  if (hsaKmtGetRuntimeCapabilities(&caps_mask) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  core::Runtime::runtime_singleton_->KfdVersion(
+      ret != HSAKMT_STATUS_NOT_SUPPORTED,
+      !!(caps_mask & HSA_RUNTIME_ENABLE_CAPS_SUPPORTS_CORE_DUMP_MASK));
+
+  if (hsaKmtGetVersion(&version_) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  if (version_.KernelInterfaceMajorVersion == kfd_version_major_min &&
+      version_.KernelInterfaceMinorVersion < kfd_version_major_min)
+    return HSA_STATUS_ERROR;
+
+  core::Runtime::runtime_singleton_->KfdVersion(version_);
+
+  if (version_.KernelInterfaceMajorVersion == 1 && version_.KernelInterfaceMinorVersion == 0)
+    core::g_use_interrupt_wait = false;
+
+  bool xnack_mode = BindXnackMode();
+  core::Runtime::runtime_singleton_->XnackEnabled(xnack_mode);
+
+  return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t KfdDriver::ShutDown() {
+  HSAKMT_STATUS ret = hsaKmtRuntimeDisable();
+  if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  ret = hsaKmtReleaseSystemProperties();
+
+  if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  return Close();
+}
 
-hsa_status_t KfdDriver::DiscoverDriver() {
-  std::unique_ptr<Driver> kfd_drv(new KfdDriver("/dev/kfd"));
+hsa_status_t KfdDriver::DiscoverDriver(std::unique_ptr<core::Driver>& driver) {
+  auto tmp_driver = std::unique_ptr<core::Driver>(new KfdDriver("/dev/kfd"));
 
-  if (kfd_drv->Open() == HSA_STATUS_SUCCESS) {
-    core::Runtime::runtime_singleton_->RegisterDriver(kfd_drv);
+  if (tmp_driver->Open() == HSA_STATUS_SUCCESS) {
+    driver = std::move(tmp_driver);
     return HSA_STATUS_SUCCESS;
   }
+
   return HSA_STATUS_ERROR;
 }
 
@@ -87,6 +127,28 @@ hsa_status_t KfdDriver::Close() {
                                                    : HSA_STATUS_ERROR;
 }
 
+hsa_status_t KfdDriver::GetSystemProperties(HsaSystemProperties& sys_props) const {
+  if (hsaKmtReleaseSystemProperties() != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  if (hsaKmtAcquireSystemProperties(&sys_props) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR;
+
+  return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t KfdDriver::GetNodeProperties(HsaNodeProperties& node_props, uint32_t node_id) const {
+  if (hsaKmtGetNodeProperties(node_id, &node_props) != HSAKMT_STATUS_SUCCESS)
+    return HSA_STATUS_ERROR;
+  return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t KfdDriver::GetEdgeProperties(std::vector<HsaIoLinkProperties>& io_link_props,
+                                          uint32_t node_id) const {
+  if (hsaKmtGetNodeIoLinkProperties(node_id, io_link_props.size(), io_link_props.data()) !=
+      HSAKMT_STATUS_SUCCESS)
+    return HSA_STATUS_ERROR;
+  return HSA_STATUS_SUCCESS;
+}
+
 hsa_status_t KfdDriver::GetAgentProperties(core::Agent &agent) const {
   return HSA_STATUS_SUCCESS;
 }
@@ -300,5 +362,33 @@ void KfdDriver::MakeKfdMemoryUnresident(const void *mem) {
   hsaKmtUnmapMemoryToGPU(const_cast<void *>(mem));
 }
 
+bool KfdDriver::BindXnackMode() {
+  // Get users' preference for Xnack mode of ROCm platform.
+  HSAint32 mode = core::Runtime::runtime_singleton_->flag().xnack();
+  bool config_xnack = (mode != Flag::XNACK_REQUEST::XNACK_UNCHANGED);
+
+  // Indicate to driver users' preference for Xnack mode
+  // Call to driver can fail and is a supported feature
+  HSAKMT_STATUS status = HSAKMT_STATUS_ERROR;
+  if (config_xnack) {
+    status = hsaKmtSetXNACKMode(mode);
+    if (status == HSAKMT_STATUS_SUCCESS) {
+      return (mode != Flag::XNACK_DISABLE);
+    }
+  }
+
+  // Get Xnack mode of devices bound by driver. This could happen
+  // when a call to SET Xnack mode fails or user has no particular
+  // preference
+  status = hsaKmtGetXNACKMode(&mode);
+  if (status != HSAKMT_STATUS_SUCCESS) {
+    debug_print(
+        "KFD does not support xnack mode query.\nROCr must assume "
+        "xnack is disabled.\n");
+    return false;
+  }
+  return (mode != Flag::XNACK_DISABLE);
+}
+
 } // namespace AMD
 } // namespace rocr

runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp

@@ -61,23 +61,19 @@ namespace AMD {
 XdnaDriver::XdnaDriver(std::string devnode_name)
     : core::Driver(core::DriverType::XDNA, devnode_name) {}
 
-XdnaDriver::~XdnaDriver() { FreeDeviceHeap(); }
-
-hsa_status_t XdnaDriver::DiscoverDriver() {
+hsa_status_t XdnaDriver::DiscoverDriver(std::unique_ptr<core::Driver>& driver) {
   const int max_minor_num(64);
   const std::string devnode_prefix("/dev/accel/accel");
 
   for (int i = 0; i < max_minor_num; ++i) {
-    std::unique_ptr<Driver> xdna_drv(
-        new XdnaDriver(devnode_prefix + std::to_string(i)));
-    if (xdna_drv->Open() == HSA_STATUS_SUCCESS) {
-      if (xdna_drv->QueryKernelModeDriver(
-              core::DriverQuery::GET_DRIVER_VERSION) == HSA_STATUS_SUCCESS) {
-        static_cast<XdnaDriver *>(xdna_drv.get())->Init();
-        core::Runtime::runtime_singleton_->RegisterDriver(xdna_drv);
+    auto tmp_driver = std::unique_ptr<Driver>(new XdnaDriver(devnode_prefix + std::to_string(i)));
+    if (tmp_driver->Open() == HSA_STATUS_SUCCESS) {
+      if (tmp_driver->QueryKernelModeDriver(core::DriverQuery::GET_DRIVER_VERSION) ==
+          HSA_STATUS_SUCCESS) {
+        driver = std::move(tmp_driver);
         return HSA_STATUS_SUCCESS;
       } else {
-        xdna_drv->Close();
+        tmp_driver->Close();
       }
     }
   }
@@ -91,6 +87,8 @@ uint64_t XdnaDriver::GetDevHeapByteSize() {
 
 hsa_status_t XdnaDriver::Init() { return InitDeviceHeap(); }
 
+hsa_status_t XdnaDriver::ShutDown() { return FreeDeviceHeap(); }
+
 hsa_status_t XdnaDriver::QueryKernelModeDriver(core::DriverQuery query) {
   switch (query) {
   case core::DriverQuery::GET_DRIVER_VERSION:
@@ -121,6 +119,25 @@ hsa_status_t XdnaDriver::Close() {
   return HSA_STATUS_SUCCESS;
 }
 
+hsa_status_t XdnaDriver::GetSystemProperties(HsaSystemProperties& sys_props) const {
+  sys_props.NumNodes = 1;
+  return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t XdnaDriver::GetNodeProperties(HsaNodeProperties& node_props, uint32_t node_id) const {
+  /// @todo XDNA driver currently only supports single-node AIE
+  /// devices over PCIe. Update this once we can get topology
+  /// information dynamically from the sysfs.
+  node_props.NumNeuralCores = 1;
+  node_props.NumIOLinks = 0;
+  return HSA_STATUS_SUCCESS;
+}
+
+hsa_status_t XdnaDriver::GetEdgeProperties(std::vector<HsaIoLinkProperties>& io_link_props,
+                                           uint32_t node_id) const {
+  return HSA_STATUS_SUCCESS;
+}
+
 hsa_status_t XdnaDriver::GetAgentProperties(core::Agent &agent) const {
   if (agent.device_type() != core::Agent::DeviceType::kAmdAieDevice) {
     return HSA_STATUS_ERROR_INVALID_AGENT;
@@ -284,8 +301,8 @@ hsa_status_t XdnaDriver::QueryDriverVersion() {
     return HSA_STATUS_ERROR;
   }
 
-  version_.major = aie_version.major;
-  version_.minor = aie_version.minor;
+  version_.KernelInterfaceMajorVersion = aie_version.major;
+  version_.KernelInterfaceMinorVersion = aie_version.minor;
 
   return HSA_STATUS_SUCCESS;
 }

runtime/hsa-runtime/core/inc/amd_aie_aql_queue.h

@@ -53,6 +53,8 @@
 namespace rocr {
 namespace AMD {
 
+class XdnaDriver;
+
 /// @brief Encapsulates HW AIE AQL Command Processor functionality. It
 /// provides the interface for things such as doorbells, queue read and
 /// write pointers, and a buffer.

runtime/hsa-runtime/core/inc/amd_available_drivers.h

@@ -0,0 +1,53 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// The University of Illinois/NCSA
+// Open Source License (NCSA)
+//
+// Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved.
+//
+// Developed by:
+//
+//                 AMD Research and AMD HSA Software Development
+//
+//                 Advanced Micro Devices, Inc.
+//
+//                 www.amd.com
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to
+// deal with the Software without restriction, including without limitation
+// the rights to use, copy, modify, merge, publish, distribute, sublicense,
+// and/or sell copies of the Software, and to permit persons to whom the
+// Software is furnished to do so, subject to the following conditions:
+//
+//  - Redistributions of source code must retain the above copyright notice,
+//    this list of conditions and the following disclaimers.
+//  - Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimers in
+//    the documentation and/or other materials provided with the distribution.
+//  - Neither the names of Advanced Micro Devices, Inc,
+//    nor the names of its contributors may be used to endorse or promote
+//    products derived from this Software without specific prior written
+//    permission.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS WITH THE SOFTWARE.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#ifndef HSA_RUNTME_CORE_INC_AMD_AVAILABLE_DRIVERS_H_
+#define HSA_RUNTME_CORE_INC_AMD_AVAILABLE_DRIVERS_H_
+
+#ifdef __linux__
+
+#include "core/inc/amd_kfd_driver.h"
+#include "core/inc/amd_xdna_driver.h"
+
+#endif
+
+#endif  // header guard

runtime/hsa-runtime/core/inc/amd_kfd_driver.h

@@ -43,6 +43,7 @@
 #ifndef HSA_RUNTIME_CORE_INC_AMD_KFD_DRIVER_H_
 #define HSA_RUNTIME_CORE_INC_AMD_KFD_DRIVER_H_
 
+#include <memory>
 #include <string>
 
 #include "hsakmt/hsakmt.h"
@@ -64,12 +65,17 @@ class KfdDriver final : public core::Driver {
 public:
   KfdDriver(std::string devnode_name);
 
-  static hsa_status_t DiscoverDriver();
+  static hsa_status_t DiscoverDriver(std::unique_ptr<core::Driver>& driver);
 
   hsa_status_t Init() override;
+  hsa_status_t ShutDown() override;
   hsa_status_t QueryKernelModeDriver(core::DriverQuery query) override;
   hsa_status_t Open() override;
   hsa_status_t Close() override;
+  hsa_status_t GetSystemProperties(HsaSystemProperties& sys_props) const override;
+  hsa_status_t GetNodeProperties(HsaNodeProperties& node_props, uint32_t node_id) const override;
+  hsa_status_t GetEdgeProperties(std::vector<HsaIoLinkProperties>& io_link_props,
+                                 uint32_t node_id) const override;
   hsa_status_t GetAgentProperties(core::Agent &agent) const override;
   hsa_status_t
   GetMemoryProperties(uint32_t node_id,
@@ -98,6 +104,17 @@ class KfdDriver final : public core::Driver {
 
   /// @brief Unpin memory.
   static void MakeKfdMemoryUnresident(const void *mem);
+
+  /// @brief Query for user preference and use that to determine Xnack mode
+  /// of ROCm system. Return true if Xnack mode is ON or false if OFF. Xnack
+  /// mode of a system is orthogonal to devices that do not support Xnack mode.
+  /// It is legal for a system with Xnack ON to have devices that do not support
+  /// Xnack functionality.
+  static bool BindXnackMode();
+
+  // Minimum acceptable KFD version numbers.
+  static const uint32_t kfd_version_major_min = 0;
+  static const uint32_t kfd_version_minor_min = 99;
 };
 
 } // namespace AMD

runtime/hsa-runtime/core/inc/amd_xdna_driver.h

@@ -129,21 +129,26 @@ inline uint32_t GetOperandCount(uint32_t arg_count) {
 class XdnaDriver final : public core::Driver {
 public:
   XdnaDriver(std::string devnode_name);
-  ~XdnaDriver();
+  ~XdnaDriver() = default;
 
-  static hsa_status_t DiscoverDriver();
+  static hsa_status_t DiscoverDriver(std::unique_ptr<core::Driver>& driver);
 
   /// @brief Returns the size of the dev heap in bytes.
   static uint64_t GetDevHeapByteSize();
 
   hsa_status_t Init() override;
+  hsa_status_t ShutDown() override;
   hsa_status_t QueryKernelModeDriver(core::DriverQuery query) override;
 
   std::unordered_map<uint32_t, void*>& GetHandleMappings();
   std::unordered_map<void*, uint32_t>& GetAddrMappings();
 
   hsa_status_t Open() override;
   hsa_status_t Close() override;
+  hsa_status_t GetSystemProperties(HsaSystemProperties& sys_props) const override;
+  hsa_status_t GetNodeProperties(HsaNodeProperties& node_props, uint32_t node_id) const override;
+  hsa_status_t GetEdgeProperties(std::vector<HsaIoLinkProperties>& io_link_props,
+                                 uint32_t node_id) const override;
   hsa_status_t GetAgentProperties(core::Agent &agent) const override;
   hsa_status_t
   GetMemoryProperties(uint32_t node_id,