1. Jeff Squyres
  2. mpi3-fortran

Commits

Jeff Squyres  committed 72347f9

Up to SVN r28941

  • Participants
  • Parent commits e03b5ab
  • Branches default

Comments (0)

Files changed (5)

File config/opal_check_pmi.m4

View file
  • Ignore whitespace
                   $3])
            ])
 
-    # restore flags - have to add CPPFLAGS so base functions can find pmi.h
-    CPPFLAGS="$opal_check_pmi_$1_save_CPPFLAGS $opal_check_pmi_$1_CPPFLAGS"
+    # restore flags
+    CPPFLAGS="$opal_check_pmi_$1_save_CPPFLAGS"
     LDFLAGS="$opal_check_pmi_$1_save_LDFLAGS"
     LIBS="$opal_check_pmi_$1_save_LIBS"
 

File ompi/mca/btl/usnic/btl_usnic_component.c

View file
  • Ignore whitespace
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 #include "opal_stdint.h"
 #include "opal/prefetch.h"
 #endif
 }
 
+static int read_device_sysfs(ompi_btl_usnic_module_t *module, const char *name)
+{
+    int ret, fd;
+    char filename[OPAL_PATH_MAX], line[256];
+
+    snprintf(filename, sizeof(filename), "/sys/class/infiniband/%s/%s",
+             ibv_get_device_name(module->device), name);
+    fd = open(filename, O_RDONLY);
+    if (fd < 0) {
+        return -1;
+    }
+
+    ret = read(fd, line, sizeof(line));
+    close(fd);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return atoi(line);
+}
+
+static int check_usnic_config(struct ibv_device_attr *device_attr,
+                              ompi_btl_usnic_module_t *module,
+                              int num_local_procs)
+{
+    char str[128];
+    int num_vfs, qp_per_vf, cq_per_vf;
+
+    /* usNIC allocates QPs as a combination of PCI virtual functions
+       (VFs) and resources inside those VFs.  Ensure that:
+
+       1. num_vfs (i.e., "usNICs") >= num_local_procs (to ensure that
+          each MPI process will be able to have its own protection
+          domain), and
+       2. num_vfs * num_qps_per_vf >= num_local_procs * NUM_CHANNELS
+          (to ensure that each MPI process will be able to get the
+          number of QPs it needs -- we know that every VF will have
+          the same number of QPs), and
+       3. num_vfs * num_cqs_per_vf >= num_local_procs * NUM_CHANNELS
+          (to ensure that each MPI process will be able to get the
+          number of CQs that it needs) */
+    num_vfs = read_device_sysfs(module, "max_vf");
+    qp_per_vf = read_device_sysfs(module, "qp_per_vf");
+    cq_per_vf = read_device_sysfs(module, "cq_per_vf");
+    if (num_vfs < 0 || qp_per_vf < 0 || cq_per_vf < 0) {
+        snprintf(str, sizeof(str), "Cannot read usNIC Linux verbs resources");
+        goto error;
+    }
+
+    if (num_vfs < num_local_procs) {
+        snprintf(str, sizeof(str), "Not enough usNICs (found %d, need %d)",
+                 num_vfs, num_local_procs);
+        goto error;
+    }
+
+    if (num_vfs * qp_per_vf < num_local_procs * USNIC_NUM_CHANNELS) {
+        snprintf(str, sizeof(str), "Not enough SQ/RQ (found %d, need %d)",
+                 num_vfs * qp_per_vf,
+                 num_local_procs * USNIC_NUM_CHANNELS);
+        goto error;
+    }
+    if (num_vfs * cq_per_vf < num_local_procs * USNIC_NUM_CHANNELS) {
+        snprintf(str, sizeof(str), "Not enough CQ per usNIC (found %d, need %d)",
+                 num_vfs * cq_per_vf,
+                 num_local_procs * USNIC_NUM_CHANNELS);
+        goto error;
+    }
+
+    /* All is good! */
+    return OMPI_SUCCESS;
+
+ error:
+    /* Sad panda */
+    opal_show_help("help-mpi-btl-usnic.txt",
+                   "not enough usnic resources",
+                   true,
+                   ompi_process_info.nodename,
+                   ibv_get_device_name(module->device),
+                   str);
+    return OMPI_ERROR;
+}
+
+
 static void
 usnic_clock_callback(int fd, short flags, void *timeout)
 {
         }
         opal_memchecker_base_mem_defined(&device_attr, sizeof(device_attr));
 
-        /* Do a little math: are there enough qp's on this device for
-           all MPI processes in this job on this server?  If not, then
-           don't even try.  We currently need 2 QPs per MPI process. */
-        if (device_attr.max_qp < num_local_procs * 2) {
-            char *str;
-            asprintf(&str, "Not enough usNIC QPs (found %d, need %d)",
-                     device_attr.max_qp, num_local_procs * 2);
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "not enough usnic resources",
-                           true,
-                           ompi_process_info.nodename,
-                           ibv_get_device_name(module->device),
-                           str);
-            free(str);
-            --mca_btl_usnic_component.num_modules;
-            --i;
-            continue;
-        } else if (device_attr.max_cq < num_local_procs * 2) {
-            char *str;
-            asprintf(&str, "Not enough usNIC CQs (found %d, need %d)",
-                     device_attr.max_cq, num_local_procs * 2);
-            opal_show_help("help-mpi-btl-usnic.txt",
-                           "not enough usnic resources",
-                           true,
-                           ompi_process_info.nodename,
-                           ibv_get_device_name(module->device),
-                           str);
-            free(str);
+        /* Check some usNIC configuration minimum settings */
+        if (check_usnic_config(&device_attr, module,
+                               num_local_procs) != OMPI_SUCCESS) {
             --mca_btl_usnic_component.num_modules;
             --i;
             continue;

File ompi/mca/pubsub/pmi/pubsub_pmi.c

View file
  • Ignore whitespace
 #include <pmi2.h>
 #endif
 
+#include "opal/mca/common/pmi/common_pmi.h"
+
 #include "ompi/info/info.h"
 #include "ompi/mca/rte/rte.h"
 #include "ompi/mca/pubsub/base/base.h"
 
 #if WANT_PMI2_SUPPORT
     if (PMI_SUCCESS != (rc = PMI2_Nameserv_publish(service_name, NULL, port_name))) {
-        OMPI_ERROR_LOG(rc);
+        OPAL_PMI_ERROR(rc, "PMI2_Nameserv_publish");
         return OMPI_ERROR;
     }
 #else
     if (PMI_SUCCESS != (rc = PMI_Publish_name(service_name, port_name))) {
-        OMPI_ERROR_LOG(rc);
+        OPAL_PMI_ERROR(rc, "PMI_Publish_name");
         return OMPI_ERROR;
     }
 #endif
 #if WANT_PMI2_SUPPORT
     port = (char*)malloc(1024*sizeof(char));  /* arbitrary size */
     if (PMI_SUCCESS != (rc = PMI2_Nameserv_lookup(service_name, NULL, port, 1024))) {
-        OMPI_ERROR_LOG(rc);
+        OPAL_PMI_ERROR(rc, "PMI2_Nameserv_lookup");
         free(port);
         return NULL;
     }
 #else
     if (PMI_SUCCESS != (rc = PMI_Lookup_name(service_name, port))) {
-        OMPI_ERROR_LOG(rc);
+        OPAL_PMI_ERROR(rc, "PMI_Lookup_name");
         return NULL;
     }
 #endif
 
 #if WANT_PMI2_SUPPORT
     if (PMI_SUCCESS != (rc = PMI2_Nameserv_unpublish(service_name, NULL))) {
-        OMPI_ERROR_LOG(rc);
+        OPAL_PMI_ERROR(rc, "PMI2_Nameserv_unpublish");
         return OMPI_ERROR;
     }
 #else
     if (PMI_SUCCESS != (rc = PMI_Unpublish_name(service_name))) {
-        OMPI_ERROR_LOG(rc);
+        OPAL_PMI_ERROR(rc, "PMI2_Nameserv_unpublish");
         return OMPI_ERROR;
     }
 #endif

File ompi/mpi/fortran/use-mpi-f08/mpi-f-interfaces-bind.h

View file
  • Ignore whitespace
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_get_elements_f
 
+subroutine ompi_get_elements_x_f(status,datatype,count,ierror) &
+   BIND(C, name="ompi_get_elements_x_f")
+   use :: mpi_f08_types, only : MPI_Status, MPI_COUNT_KIND
+   implicit none
+   TYPE(MPI_Status), INTENT(IN) :: status
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(MPI_COUNT_KIND), INTENT(OUT) :: count
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_get_elements_x_f
+
 subroutine ompi_pack_f(inbuf,incount,datatype,outbuf,outsize, &
                        position,comm,ierror) &
    BIND(C, name="ompi_pack_f")
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_type_get_extent_f
 
+subroutine ompi_type_get_extent_x_f(datatype,lb,extent,ierror) &
+   BIND(C, name="ompi_type_get_extent_x_f")
+   use :: mpi_f08_types, only : MPI_COUNT_KIND
+   implicit none
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(MPI_COUNT_KIND), INTENT(OUT) :: lb, extent
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_type_get_extent_x_f
+
 subroutine ompi_type_get_true_extent_f(datatype,true_lb,true_extent,ierror) &
    BIND(C, name="ompi_type_get_true_extent_f")
    use :: mpi_f08_types, only : MPI_ADDRESS_KIND
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_type_get_true_extent_f
 
+subroutine ompi_type_get_true_extent_x_f(datatype,true_lb,true_extent,ierror) &
+   BIND(C, name="ompi_type_get_true_extent_x_f")
+   use :: mpi_f08_types, only : MPI_COUNT_KIND
+   implicit none
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(MPI_COUNT_KIND), INTENT(OUT) :: true_lb, true_extent
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_type_get_true_extent_x_f
+
 subroutine ompi_type_indexed_f(count,array_of_blocklengths, &
                                array_of_displacements,oldtype,newtype,ierror) &
    BIND(C, name="ompi_type_indexed_f")
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_type_size_f
 
+subroutine ompi_type_size_x_f(datatype,size,ierror) &
+   BIND(C, name="ompi_type_size_x_f")
+   use :: mpi_f08_types, only : MPI_COUNT_KIND
+   implicit none
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(MPI_COUNT_KIND), INTENT(OUT) :: size
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_type_size_x_f
+
 subroutine ompi_type_vector_f(count,blocklength,stride,oldtype,newtype,ierror) &
    BIND(C, name="ompi_type_vector_f")
    implicit none
    INTEGER, INTENT(OUT) :: ierror
 end subroutine ompi_status_set_elements_f
 
+subroutine ompi_status_set_elements_x_f(status,datatype,count,ierror) &
+   BIND(C, name="ompi_status_set_elements_x_f")
+   use :: mpi_f08_types, only : MPI_Status, MPI_COUNT_KIND
+   implicit none
+   TYPE(MPI_Status), INTENT(INOUT) :: status
+   INTEGER, INTENT(IN) :: datatype
+   INTEGER(MPI_COUNT_KIND), INTENT(IN) :: count
+   INTEGER, INTENT(OUT) :: ierror
+end subroutine ompi_status_set_elements_x_f
+
 #if OMPI_PROVIDE_MPI_FILE_INTERFACE
 
 subroutine ompi_file_close_f(fh,ierror) &

File ompi/runtime/ompi_mpi_params.c

View file
  • Ignore whitespace
 bool ompi_mpi_leave_pinned_pipeline = false;
 bool ompi_have_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
 bool ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
-bool ompi_mpi_cuda_support = OPAL_INT_TO_BOOL(OMPI_CUDA_SUPPORT);
+bool ompi_mpi_built_with_cuda_support = OPAL_INT_TO_BOOL(OMPI_CUDA_SUPPORT);
+bool ompi_mpi_cuda_support;
 
 bool ompi_mpi_yield_when_idle = true;
 int ompi_mpi_event_tick_rate = -1;
         ompi_use_sparse_group_storage = false;
     }
 
-    ompi_mpi_cuda_support = !!(OMPI_CUDA_SUPPORT);
+    (void) mca_base_var_register("ompi", "mpi", NULL, "built_with_cuda_support",
+                                 "Whether CUDA GPU buffer support is built into library or not",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
+                                 MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
+                                 OPAL_INFO_LVL_4,
+                                 MCA_BASE_VAR_SCOPE_CONSTANT,
+                                 &ompi_mpi_built_with_cuda_support);
+
+	/* Current default is to enable CUDA support if it is built into library */
+	ompi_mpi_cuda_support = ompi_mpi_built_with_cuda_support;
+
     (void) mca_base_var_register("ompi", "mpi", NULL, "cuda_support",
                                  "Whether CUDA GPU buffer support is enabled or not",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0,
-                                 !!(OMPI_CUDA_SUPPORT) ? 0 : MCA_BASE_VAR_FLAG_DEFAULT_ONLY,
-                                 OPAL_INFO_LVL_9,
-                                 !!(OMPI_CUDA_SUPPORT) ? MCA_BASE_VAR_SCOPE_READONLY : MCA_BASE_VAR_SCOPE_CONSTANT,
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_4,
+                                 MCA_BASE_VAR_SCOPE_LOCAL,
                                  &ompi_mpi_cuda_support);
-    if (ompi_mpi_cuda_support && !(OMPI_CUDA_SUPPORT)) {
+    if (ompi_mpi_cuda_support && !ompi_mpi_built_with_cuda_support) {
         opal_show_help("help-mpi-runtime.txt", "no cuda support",
                        true);
-        ompi_mpi_cuda_support = false;
+        ompi_rte_abort(1, NULL);
     }
 
     return OMPI_SUCCESS;