Separate kernel, pll, memory interfaces for different devices

Author: sophimao

include/acl_hal_mmd.h

@@ -149,6 +149,14 @@ struct aocl_mmd_streaming_kernel_arg_info_t {
   std::vector<char> value;
 };
 
+// MMD interface ID cache populated by calling aocl_mmd_get_info with
+// corresponding interface MMD info ID
+typedef struct {
+  int kernel_interface = -1;
+  int pll_interface = -1;
+  int memory_interface = -1;
+} aocl_mmd_interface_info_t;
+
 // MMD Version checking
 // Since MMD version changes only with major releases it is safe to assume
 // this is a float with at most one decimal
@@ -286,6 +294,7 @@ typedef struct {
   int handle;
   std::string name;
   acl_mmd_dispatch_t *mmd_dispatch;
+  aocl_mmd_interface_info_t mmd_ifaces;
 } acl_mmd_device_t;
 
 #endif

src/acl_hal_mmd.cpp

@@ -247,11 +247,6 @@ static size_t num_board_pkgs;
 static unsigned num_physical_devices = 0;
 static double min_MMD_version = DBL_MAX;
 
-// Interfaces
-static int kernel_interface = -1;
-static int pll_interface = -1;
-static int memory_interface = -1;
-
 static acl_bsp_io bsp_io_kern[ACL_MAX_DEVICE];
 static acl_bsp_io bsp_io_pll[ACL_MAX_DEVICE];
 static acl_kernel_if kern[ACL_MAX_DEVICE];
@@ -1159,6 +1154,9 @@ static int l_try_device(unsigned int physical_device_id, const char *name,
   }
 
   // Get interfaces - for now assume one kernel, one pll and one memory
+  int kernel_interface = -1;
+  int pll_interface = -1;
+  int memory_interface = -1;
   ACL_HAL_DEBUG_MSG_VERBOSE(1,
                             "HAL : Getting interfaces via aocl_mmd_get_info\n");
   device->mmd_dispatch->aocl_mmd_get_info(device->handle,
@@ -1182,22 +1180,23 @@ static int l_try_device(unsigned int physical_device_id, const char *name,
     printf("Error mmd_get_info: handles for kernel, pll, and memory must be "
            "unique and greater than 0\n");
   }
+  device->mmd_ifaces.kernel_interface = kernel_interface;
+  device->mmd_ifaces.pll_interface = pll_interface;
+  device->mmd_ifaces.memory_interface = memory_interface;
+
   device->mmd_dispatch->aocl_mmd_set_status_handler(
       device->handle, acl_hal_mmd_status_handler, NULL);
 
   kern[physical_device_id].physical_device_id = physical_device_id;
 
-  bsp_io_kern[physical_device_id].debug_verbosity = debug_verbosity;
-
   // Initialize PLL
   if (pll_interface >= 0) {
     bsp_io_pll[physical_device_id].device_info = device;
     bsp_io_pll[physical_device_id].read = acl_pll_read;
     bsp_io_pll[physical_device_id].write = acl_pll_write;
     bsp_io_pll[physical_device_id].get_time_ns = acl_bsp_get_timestamp;
     bsp_io_pll[physical_device_id].printf = printf;
-    bsp_io_pll[physical_device_id].debug_verbosity =
-        bsp_io_kern[physical_device_id].debug_verbosity;
+    bsp_io_pll[physical_device_id].debug_verbosity = debug_verbosity;
     info_assert(acl_pll_init(&pll[physical_device_id],
                              bsp_io_pll[physical_device_id], "") == 0,
                 "Failed to read PLL config");
@@ -1215,6 +1214,7 @@ static int l_try_device(unsigned int physical_device_id, const char *name,
   bsp_io_kern[physical_device_id].write = acl_kernel_if_write;
   bsp_io_kern[physical_device_id].get_time_ns = acl_bsp_get_timestamp;
   bsp_io_kern[physical_device_id].printf = printf;
+  bsp_io_kern[physical_device_id].debug_verbosity = debug_verbosity;
 
   info_assert(acl_kernel_if_init(&kern[physical_device_id],
                                  bsp_io_kern[physical_device_id], sys) == 0,
@@ -1892,7 +1892,7 @@ int acl_hal_mmd_program_device(unsigned int physical_device_id,
   device_info[physical_device_id].mmd_dispatch->aocl_mmd_set_status_handler(
       device_info[physical_device_id].handle, acl_hal_mmd_status_handler, NULL);
   acl_kernel_if_update(devdef->autodiscovery_def, &kern[physical_device_id]);
-  if (pll_interface >= 0) {
+  if (device_info[physical_device_id].mmd_ifaces.pll_interface >= 0) {
     if (acl_pkg_section_exists(binary, ACL_PKG_SECTION_PLL_CONFIG,
                                &pll_config_len)) {
       info_assert(acl_pll_init(&pll[physical_device_id],
@@ -2099,7 +2099,8 @@ void acl_hal_mmd_copy_hostmem_to_globalmem(cl_event event, const void *src,
 
   s = device_info[physical_device_id].mmd_dispatch->aocl_mmd_write(
       device_info[physical_device_id].handle, (aocl_mmd_op_t)event, size, src,
-      memory_interface, (size_t)ACL_STRIP_PHYSICAL_ID(dest));
+      device_info[physical_device_id].mmd_ifaces.memory_interface,
+      (size_t)ACL_STRIP_PHYSICAL_ID(dest));
   assert(s == 0 && "mmd read/write failed");
 }
 
@@ -2122,7 +2123,8 @@ void acl_hal_mmd_copy_globalmem_to_hostmem(cl_event event, const void *src,
 
   s = device_info[physical_device_id].mmd_dispatch->aocl_mmd_read(
       device_info[physical_device_id].handle, (aocl_mmd_op_t)event, size, dest,
-      memory_interface, (size_t)ACL_STRIP_PHYSICAL_ID(src));
+      device_info[physical_device_id].mmd_ifaces.memory_interface,
+      (size_t)ACL_STRIP_PHYSICAL_ID(src));
   assert(s == 0 && "mmd read/write failed");
 }
 
@@ -2150,7 +2152,8 @@ void acl_hal_mmd_copy_globalmem_to_globalmem(cl_event event, const void *src,
     // Let the MMD provider do the intra-device copy.
     s = device_info[physical_device_id_src].mmd_dispatch->aocl_mmd_copy(
         device_info[physical_device_id_src].handle, (aocl_mmd_op_t)event, size,
-        memory_interface, (size_t)ACL_STRIP_PHYSICAL_ID(src),
+        device_info[physical_device_id_src].mmd_ifaces.memory_interface,
+        (size_t)ACL_STRIP_PHYSICAL_ID(src),
         (size_t)ACL_STRIP_PHYSICAL_ID(dest));
   } else {
     // Copy from device to device via host memory.
@@ -2174,7 +2177,9 @@ void acl_hal_mmd_copy_globalmem_to_globalmem(cl_event event, const void *src,
     // Read the initial block into data[0]
     s = device_info[physical_device_id_src].mmd_dispatch->aocl_mmd_read(
         device_info[physical_device_id_src].handle, NULL, transfer_size,
-        &data[0][0], memory_interface, (size_t)ACL_STRIP_PHYSICAL_ID(src));
+        &data[0][0],
+        device_info[physical_device_id_src].mmd_ifaces.memory_interface,
+        (size_t)ACL_STRIP_PHYSICAL_ID(src));
     src = (const char *)src + transfer_size;
     size -= transfer_size;
 
@@ -2198,13 +2203,15 @@ void acl_hal_mmd_copy_globalmem_to_globalmem(cl_event event, const void *src,
 
       device_info[physical_device_id_dst].mmd_dispatch->aocl_mmd_write(
           device_info[physical_device_id_dst].handle, &dst_dev_done,
-          transfer_size, &data[buffer][0], memory_interface,
+          transfer_size, &data[buffer][0],
+          device_info[physical_device_id_dst].mmd_ifaces.memory_interface,
           (size_t)ACL_STRIP_PHYSICAL_ID(dest));
 
       if (transfer_size_next)
         device_info[physical_device_id_src].mmd_dispatch->aocl_mmd_read(
             device_info[physical_device_id_src].handle, &src_dev_done,
-            transfer_size_next, &data[1 - buffer][0], memory_interface,
+            transfer_size_next, &data[1 - buffer][0],
+            device_info[physical_device_id_src].mmd_ifaces.memory_interface,
             (size_t)ACL_STRIP_PHYSICAL_ID(src));
       else
         src_dev_done = 1;
@@ -2787,7 +2794,7 @@ static size_t acl_kernel_if_read(acl_bsp_io *io, dev_addr_t src, char *dest,
                             size, (size_t)src, (size_t)dest);
   return io->device_info->mmd_dispatch->aocl_mmd_read(
              io->device_info->handle, NULL, size, (void *)dest,
-             kernel_interface, (size_t)src) == 0
+             io->device_info->mmd_ifaces.kernel_interface, (size_t)src) == 0
              ? size
              : 0;
 }
@@ -2800,7 +2807,7 @@ static size_t acl_kernel_if_write(acl_bsp_io *io, dev_addr_t dest,
                             size, (size_t)src, (size_t)dest);
   return io->device_info->mmd_dispatch->aocl_mmd_write(
              io->device_info->handle, NULL, size, (const void *)src,
-             kernel_interface, (size_t)dest) == 0
+             io->device_info->mmd_ifaces.kernel_interface, (size_t)dest) == 0
              ? size
              : 0;
 }
@@ -2812,8 +2819,8 @@ static size_t acl_pll_read(acl_bsp_io *io, dev_addr_t src, char *dest,
   ACL_HAL_DEBUG_MSG_VERBOSE(5, "HAL Reading from PLL: %zu bytes %zx -> %zx\n",
                             size, (size_t)src, (size_t)dest);
   return io->device_info->mmd_dispatch->aocl_mmd_read(
-             io->device_info->handle, NULL, size, (void *)dest, pll_interface,
-             (size_t)src) == 0
+             io->device_info->handle, NULL, size, (void *)dest,
+             io->device_info->mmd_ifaces.pll_interface, (size_t)src) == 0
              ? size
              : 0;
 }
@@ -2826,7 +2833,7 @@ static size_t acl_pll_write(acl_bsp_io *io, dev_addr_t dest, const char *src,
                             size, (size_t)src, (size_t)dest);
   return io->device_info->mmd_dispatch->aocl_mmd_write(
              io->device_info->handle, NULL, size, (const void *)src,
-             pll_interface, (size_t)dest) == 0
+             io->device_info->mmd_ifaces.pll_interface, (size_t)dest) == 0
              ? size
              : 0;
 }
@@ -3079,14 +3086,16 @@ size_t acl_hal_mmd_read_csr(unsigned int physical_device_id, uintptr_t offset,
                             void *ptr, size_t size) {
   return device_info[physical_device_id].mmd_dispatch->aocl_mmd_read(
       device_info[physical_device_id].handle, NULL, size, (void *)ptr,
-      kernel_interface, (size_t)offset);
+      device_info[physical_device_id].mmd_ifaces.kernel_interface,
+      (size_t)offset);
 }
 
 size_t acl_hal_mmd_write_csr(unsigned int physical_device_id, uintptr_t offset,
                              const void *ptr, size_t size) {
   return device_info[physical_device_id].mmd_dispatch->aocl_mmd_write(
       device_info[physical_device_id].handle, NULL, size, (const void *)ptr,
-      kernel_interface, (size_t)offset);
+      device_info[physical_device_id].mmd_ifaces.kernel_interface,
+      (size_t)offset);
 }
 
 void acl_hal_mmd_simulation_streaming_kernel_start(