opal/mca/common: match efa nic by device name

EFA devices do not have SysImageGUID or NodeGUID, and therefore cannot
be correctly matched with those identifiers. As a result, EFA devices
are being selected in round robin regardless of affinity to the running
process.

This patch introduces a logical branch to match EFA nics using device
name. With this change the nearest EFA nic is selected relative to the
running process.

Signed-off-by: Wenduo Wang <wenduwan@amazon.com>

Author: wenduwan

opal/mca/common/ofi/common_ofi.c

@@ -589,6 +589,20 @@ get_nearest_nics(int *num_distances, pmix_value_t **valin)
 }
 
 #if OPAL_OFI_PCI_DATA_AVAILABLE
+static bool match_device_by_name_prefix(pmix_device_distance_t *distances, int num_distances, char *domain_name)
+{
+    char *dev_name = NULL;
+
+    for (int i = 0; i < num_distances; ++i) {
+        dev_name = distances[i].osname;
+        if (dev_name && 0 == strncasecmp(dev_name, domain_name, MIN(strlen(dev_name), strlen(domain_name)))) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 /**
  * Determine if a device is nearest
  *
@@ -606,31 +620,19 @@ get_nearest_nics(int *num_distances, pmix_value_t **valin)
  *
  */
 #if HWLOC_API_VERSION < 0x00020000
-static bool is_near(pmix_device_distance_t *distances,
-                    int num_distances,
-                    hwloc_topology_t topology,
-                    struct fi_pci_attr pci)
+static bool match_device_by_pci_bus_id(pmix_device_distance_t *distances, int num_distances,
+                                       struct fi_pci_attr pci)
 {
     return true;
 }
 #else
-static bool is_near(pmix_device_distance_t *distances,
-                    int num_distances,
-                    hwloc_topology_t topology,
-                    struct fi_pci_attr pci)
+static bool match_device_by_pci_bus_id(pmix_device_distance_t *distances, int num_distances,
+                                       struct fi_pci_attr pci)
 {
     hwloc_obj_t pcidev, osdev;
 
-    /* if we failed to find any distances, then we consider all interfaces
-     * to be of equal distances and let the caller decide how to handle
-     * them
-     */
-    if (!distances)
-        return true;
-
-    pcidev = hwloc_get_pcidev_by_busid(topology, pci.domain_id,
-                        pci.bus_id, pci.device_id,
-                        pci.function_id);
+    pcidev = hwloc_get_pcidev_by_busid(opal_hwloc_topology, pci.domain_id, pci.bus_id,
+                                       pci.device_id, pci.function_id);
     if (!pcidev)
         return false;
 
@@ -675,8 +677,31 @@ static bool is_near(pmix_device_distance_t *distances,
 
     return false;
 }
-#endif
-#endif  // OPAL_OFI_PCI_DATA_AVAILABLE
+#endif /* HWLOC_API_VERSION < 0x00020000 */
+
+static bool is_near(pmix_device_distance_t *distances,
+                    int num_distances,
+                    struct fi_info *fi_info)
+{
+    /* if we failed to find any distances, then we consider all interfaces
+     * to be of equal distances and let the caller decide how to handle
+     * them
+     */
+    if (!distances)
+        return true;
+
+    if (fi_info && fi_info->fabric_attr && fi_info->fabric_attr->prov_name
+        && 0 == strcasecmp("efa", fi_info->fabric_attr->prov_name)) {
+        return match_device_by_name_prefix(distances, num_distances, fi_info->domain_attr->name);
+    } else if (fi_info && fi_info->nic && fi_info->nic->bus_attr
+               && fi_info->nic->bus_attr->bus_type == FI_BUS_PCI) {
+        return match_device_by_pci_bus_id(distances, num_distances,
+                                          fi_info->nic->bus_attr->attr.pci);
+    }
+
+    return false;
+}
+#endif /* OPAL_OFI_PCI_DATA_AVAILABLE */
 
 /* Count providers returns the number of providers present in an fi_info list
  *     @param (IN) provider_list    struct fi_info* list of providers available
@@ -789,7 +814,6 @@ struct fi_info *opal_common_ofi_select_provider(struct fi_info *provider_list,
 #if OPAL_OFI_PCI_DATA_AVAILABLE
     pmix_device_distance_t *distances = NULL;
     pmix_value_t *pmix_val;
-    struct fi_pci_attr pci;
     int num_distances = 0;
 #endif
     bool near;
@@ -828,15 +852,10 @@ struct fi_info *opal_common_ofi_select_provider(struct fi_info *provider_list,
     /* Cycle through remaining fi_info objects, looking for alike providers */
     while (NULL != current_provider) {
         if (!check_provider_attr(provider, current_provider)) {
-            near = false;
 #if OPAL_OFI_PCI_DATA_AVAILABLE
-            if (NULL != current_provider->nic
-                && NULL != current_provider->nic->bus_attr
-                && current_provider->nic->bus_attr->bus_type == FI_BUS_PCI) {
-                pci = current_provider->nic->bus_attr->attr.pci;
-                near = is_near(distances, num_distances,
-                               opal_hwloc_topology, pci);
-            }
+            near = is_near(distances, num_distances, current_provider);
+#else
+            near = false;
 #endif
             /* We could have multiple near providers */
             if (near && !provider_found) {
@@ -865,13 +884,7 @@ struct fi_info *opal_common_ofi_select_provider(struct fi_info *provider_list,
     }
 
 #if OPAL_OFI_PCI_DATA_AVAILABLE
-    if (NULL != provider->nic
-        && NULL != provider->nic->bus_attr
-        && provider->nic->bus_attr->bus_type == FI_BUS_PCI) {
-        pci = provider->nic->bus_attr->attr.pci;
-        near = is_near(distances, num_distances,
-                       opal_hwloc_topology, pci);
-    }
+    near = is_near(distances, num_distances, provider);
 #endif
 
 #if OPAL_ENABLE_DEBUG