Source

mpi3-fortran / ompi / mca / mtl / psm / mtl_psm_component.c

/*
 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2005 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2006-2010 QLogic Corporation. All rights reserved.
 * Copyright (c) 2012      Los Alamos National Security, LLC.
 *                         All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#include "ompi_config.h"

#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/mca/base/mca_base_param.h"
#include "ompi/proc/proc.h"

#include "mtl_psm.h"
#include "mtl_psm_types.h"
#include "mtl_psm_request.h"

#include "psm.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

static int ompi_mtl_psm_component_open(void);
static int ompi_mtl_psm_component_close(void);
static int ompi_mtl_psm_component_register(void);

static mca_mtl_base_module_t* ompi_mtl_psm_component_init( bool enable_progress_threads, 
                                                          bool enable_mpi_threads );

mca_mtl_psm_component_t mca_mtl_psm_component = {

    {
        /* First, the mca_base_component_t struct containing meta
         * information about the component itself */
        
        {
            MCA_MTL_BASE_VERSION_2_0_0,
            
            "psm", /* MCA component name */
            OMPI_MAJOR_VERSION,  /* MCA component major version */
            OMPI_MINOR_VERSION,  /* MCA component minor version */
            OMPI_RELEASE_VERSION,  /* MCA component release version */
            ompi_mtl_psm_component_open,  /* component open */
            ompi_mtl_psm_component_close,  /* component close */
	    NULL,
	    ompi_mtl_psm_component_register
        },
        {
            /* The component is not checkpoint ready */
            MCA_BASE_METADATA_PARAM_NONE
        },
        
        ompi_mtl_psm_component_init  /* component init */
    }
};

    
static int
ompi_mtl_psm_component_register(void)
{
    int value;
    char *service_id = NULL;
    char *path_res = NULL;
    
    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, 
			   "connect_timeout",
			   "PSM connection timeout value in seconds",
			   false, false, 180, &ompi_mtl_psm.connect_timeout);
  
    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, 
			   "debug",
			   "PSM debug level",
			   false, false, 1, 
			   &value);
    ompi_mtl_psm.debug_level = value;
  
    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, 
			   "ib_unit",
			   "Truescale unit to use",
			   false, false, -1, 
			   &ompi_mtl_psm.ib_unit);

    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, 
			   "ib_port",
			   "Truescale port on unit to use",
			   false, false, 0, 
			   &ompi_mtl_psm.ib_port);

    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, 
			   "ib_service_level",
			   "Infiniband service level"
			   "(0 <= SL <= 15)",
			   false, false, 0, &ompi_mtl_psm.ib_service_level);
  
    ompi_mtl_psm.ib_pkey = 0x7fffUL;
    mca_base_param_reg_int(&mca_mtl_psm_component.super.mtl_version, 
			   "ib_pkey",
			   "Infiniband partition key",
			   false, false, 0x7fffUL, 
			   &value);
    ompi_mtl_psm.ib_pkey = value;
    
#if PSM_VERNO >= 0x010d
    mca_base_param_reg_string(&mca_mtl_psm_component.super.mtl_version,
			      "ib_service_id",
			      "Infiniband service ID to use for application (default is 0)",
			      false, false, "0x1000117500000000",
			      &service_id);
    ompi_mtl_psm.ib_service_id = (uint64_t) strtoull(service_id, NULL, 0);
    
    mca_base_param_reg_string(&mca_mtl_psm_component.super.mtl_version,
			      "path_query",
			      "Path record query mechanisms (valid values: opp, none)",
			      false, false, NULL, &path_res);
    if ((NULL != path_res) && strcasecmp(path_res, "none")) {
      if (!strcasecmp(path_res, "opp"))
	ompi_mtl_psm.path_res_type = PSM_PATH_RES_OPP;
      else {
	ompi_show_help("help-mtl-psm.txt",
		       "path query mechanism unknown", true,
		       path_res, "OfedPlus (opp) | Static Routes (none)");
	return OMPI_ERR_NOT_FOUND;
      }
    }
    else {
      /* Default is "static/none" path record queries */
      ompi_mtl_psm.path_res_type = PSM_PATH_RES_NONE;
    }
#endif
    
    if (ompi_mtl_psm.ib_service_level < 0)  {
      ompi_mtl_psm.ib_service_level = 0;
    } else if (ompi_mtl_psm.ib_service_level > 15) {
      ompi_mtl_psm.ib_service_level = 15;
    }
  
  return OMPI_SUCCESS;
    
}

static int
ompi_mtl_psm_component_open(void)
{
  struct stat st;
  
  /* Component available only if Truescale hardware is present */
  if (0 == stat("/dev/ipath", &st)) {
    return OMPI_SUCCESS;
  }
  else {
    return OPAL_ERR_NOT_AVAILABLE;
  }
}

static int
ompi_mtl_psm_component_close(void)
{
    return OMPI_SUCCESS;
}

static int
get_num_local_procs(int *out_nlp)
{
    /* num_local_peers does not include us in
     * its calculation, so adjust for that */
    *out_nlp = (int)(1 + ompi_process_info.num_local_peers);
    return OMPI_SUCCESS;
}

static int
get_local_rank(int *out_rank)
{
    orte_node_rank_t my_node_rank;

    *out_rank = 0;

    if (ORTE_NODE_RANK_INVALID == (my_node_rank =
        ompi_process_info.my_node_rank)) {
        return OMPI_ERROR;
    }
    *out_rank = (int)my_node_rank;
    return OMPI_SUCCESS;
}

static mca_mtl_base_module_t *
ompi_mtl_psm_component_init(bool enable_progress_threads,
                            bool enable_mpi_threads)
{
    psm_error_t	err;
    int	verno_major = PSM_VERNO_MAJOR;
    int verno_minor = PSM_VERNO_MINOR;
    int local_rank = -1, num_local_procs = 0;

    /* Compute the total number of processes on this host and our local rank
     * on that node. We need to provide PSM with these values so it can 
     * allocate hardware contexts appropriately across processes.
     */
    if (OMPI_SUCCESS != get_num_local_procs(&num_local_procs)) {
        opal_output(0, "Cannot determine number of local processes. "
                    "Cannot continue.\n");
        return NULL;
    }
    if (OMPI_SUCCESS != get_local_rank(&local_rank)) {
        opal_output(0, "Cannot determine local rank. Cannot continue.\n");
        return NULL;
    }
     
    err = psm_error_register_handler(NULL /* no ep */,
			             PSM_ERRHANDLER_NOP);
    if (err) {
        opal_output(0, "Error in psm_error_register_handler (error %s)\n", 
		    psm_error_get_string(err));
	return NULL;
    }
    
#if PSM_VERNO >= 0x010c
    /* Set infinipath debug level */
    err = psm_setopt(PSM_COMPONENT_CORE, 0, PSM_CORE_OPT_DEBUG, 
		     (const void*) &ompi_mtl_psm.debug_level, 
		     sizeof(unsigned));
    if (err) {
      /* Non fatal error. Can continue */
      ompi_show_help("help-mtl-psm.txt",
		     "psm init", false,
		     psm_error_get_string(err));
    }
#endif
    
    /* Only allow for shm and ipath devices in 2.0 and earlier releases 
     * (unless the user overrides the setting).
     */
    
    if (PSM_VERNO >= 0x0104) {
      setenv("PSM_DEVICES", "self,shm,ipath", 0);
    }
    else {
      setenv("PSM_DEVICES", "shm,ipath", 0);
    }
    
    err = psm_init(&verno_major, &verno_minor);
    if (err) {
      ompi_show_help("help-mtl-psm.txt",
		     "psm init", true,
		     psm_error_get_string(err));
      return NULL;
    }
    
    /* Complete PSM initialization */
    ompi_mtl_psm_module_init(local_rank, num_local_procs);

    ompi_mtl_psm.super.mtl_request_size = 
      sizeof(mca_mtl_psm_request_t) - 
      sizeof(struct mca_mtl_request_t);
    
    return &ompi_mtl_psm.super;
}
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.