Commits

Anonymous committed 2fd9abf

Add support for autodetecting a MLNX HCA in the rmaps min distance feature. In this way, .ini files distributed with software stacks need not specify a particular HCA but instead may select the key word auto which will automatically select the discovered device. To use this feature, simply pass the keyword auto instead of a specific device name, --mca rmaps_base_dist_hca auto. If more than one card is installed, the mapper will inform the user of this and, at this point, the user will then need to specify which card via the normal route, e.g. --mca rmaps_base_dist_hca <dev_name>. This should be added to \ncmr=v1.7.4:reviewer=rhc:subject=Autodetect logic for min dist mapping

Comments (0)

Files changed (4)

opal/mca/hwloc/base/base.h

                                                        hwloc_obj_t obj,
                                                        opal_hwloc_resource_type_t rtype);
 
-OPAL_DECLSPEC void opal_hwloc_get_sorted_numa_list(hwloc_topology_t topo, 
-                                    const char* device_name, 
+OPAL_DECLSPEC int opal_hwloc_get_sorted_numa_list(hwloc_topology_t topo, 
+                                    char* device_name, 
                                     opal_list_t *sorted_list);
 
 /**

opal/mca/hwloc/base/hwloc_base_util.c

     }
 }
 
-static void sort_by_dist(hwloc_topology_t topo, const char* device_name, opal_list_t *sorted_list)
+static void sort_by_dist(hwloc_topology_t topo, char* device_name, opal_list_t *sorted_list)
 {
     hwloc_obj_t device_obj = NULL;
     hwloc_obj_t obj = NULL, root = NULL;
                     obj = obj->parent;
                 }
                 if (obj == NULL) {
+                    opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
+                            "hwloc:base:get_sorted_numa_list: NUMA node closest to %s wasn't found.",
+                            device_name);
                     return;
                 } else {
                     close_node_index = obj->logical_index;
                     /* we can try to find distances under group object. This info can be there. */
                     depth = hwloc_get_type_depth(topo, HWLOC_OBJ_NODE);
                     if (depth < 0) {
+                        opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
+                                "hwloc:base:get_sorted_numa_list: There is no information about distances on the node.");
                         return;
                     }
                     root = hwloc_get_root_obj(topo);
                 }
                 /* find all distances for our close node with logical index = close_node_index as close_node_index + nbobjs*j */
                 if ((NULL == distances) || (0 == distances->nbobjs)) {
+                    opal_output_verbose(5, opal_hwloc_base_framework.framework_output,
+                            "hwloc:base:get_sorted_numa_list: There is no information about distances on the node.");
                     return;
                 }
                 /* fill list of numa nodes */
     }
 }
 
-void opal_hwloc_get_sorted_numa_list(hwloc_topology_t topo, const char* device_name, opal_list_t *sorted_list)
+static int find_devices(hwloc_topology_t topo, char* device_name) 
+{
+    hwloc_obj_t device_obj = NULL;
+    int count = 0;
+    for (device_obj = hwloc_get_obj_by_type(topo, HWLOC_OBJ_OS_DEVICE, 0); device_obj; device_obj = hwloc_get_next_osdev(topo, device_obj)) {
+        if (device_obj->attr->osdev.type == HWLOC_OBJ_OSDEV_OPENFABRICS) {
+            count++;
+            free(device_name);
+            device_name = strdup(device_obj->name);
+        }
+    }
+    return count;
+}
+
+int opal_hwloc_get_sorted_numa_list(hwloc_topology_t topo, char* device_name, opal_list_t *sorted_list)
 {
     hwloc_obj_t obj;
     opal_list_item_t *item;
     opal_hwloc_summary_t *sum;
     opal_hwloc_topo_data_t *data;
     orte_rmaps_numa_node_t *numa, *copy_numa;
+    int count;
 
     obj = hwloc_get_root_obj(topo);
 
                         copy_numa->dist_from_closed = numa->dist_from_closed;
                         opal_list_append(sorted_list, &copy_numa->super);
                     }
-                    return;
+                    return 0;
                 }else {
                     /* don't already know it - go get it */
+                    /* firstly we check if we need to autodetect OpenFabrics  devices or we have the specified one */
+                    if (!strcmp(device_name, "auto")) {
+                        count = find_devices(topo, device_name);
+                       if (count > 1) {
+                           return count;
+                       }
+                    }
+                    if (!device_name || (strlen(device_name) == 0)) {
+                        return 1;
+                    }
                     sort_by_dist(topo, device_name, sorted_list);
                     /* store this info in summary object for later usage */
                     OPAL_LIST_FOREACH(numa, sorted_list, orte_rmaps_numa_node_t) {
                         copy_numa->dist_from_closed = numa->dist_from_closed;
                         opal_list_append(&(sum->sorted_by_dist_list), &copy_numa->super);
                     }
-                    return;
+                    return 0;
                 }
             }
         }

orte/mca/rmaps/mindist/help-orte-rmaps-md.txt

   Node: %s
 
 Open MPI therefore cannot mapp the application as specified.
+#
+[orte-rmaps-mindist:several-hca-devices]
+There are several OpenFabrics devices found on at least one node. Please specify the definite one.
+
+  Devices: %d
+  Node: %s
+
+Open MPI therefore cannot mapp the application as specified.

orte/mca/rmaps/mindist/rmaps_mindist_module.c

     mca_base_component_t *c = &mca_rmaps_mindist_component.base_version;
     bool initial_map=true;
     bool bynode = false;
+    int ret;
 
     /* this mapper can only handle initial launch
      * when mindist mapping is desired
              * so we call opal_hwloc_base_get_nbobjs_by_type */
             opal_hwloc_base_get_nbobjs_by_type(node->topology, HWLOC_OBJ_NODE, 0, OPAL_HWLOC_AVAILABLE);
             OBJ_CONSTRUCT(&numa_list, opal_list_t);
-            opal_hwloc_get_sorted_numa_list(node->topology, orte_rmaps_base.device, &numa_list);
+            ret = opal_hwloc_get_sorted_numa_list(node->topology, orte_rmaps_base.device, &numa_list);
+            if (ret > 1) {
+                orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:several-hca-devices",
+                        true, ret, node->name);
+                rc = ORTE_ERR_SILENT;
+                goto error;
+            }
             if (opal_list_get_size(&numa_list) > 0) {
                 j = 0;
                 required = 0;
         }
         OBJ_DESTRUCT(&node_list);
     }
-
+    free(orte_rmaps_base.device);
     return ORTE_SUCCESS;
 
 error:
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.