From aba9933e4f9a0a2e5806635f381536bcb6b2906a Mon Sep 17 00:00:00 2001
From: "Paul H. Hargrove" <PHHargrove@lbl.gov>
Date: Mon, 10 Dec 2018 19:33:28 -0800
Subject: [PATCH] ibv: add support for non-default IB Partition Key
This commit implements and documents environment variable
"GASNET_IBV_PKEY" to allow selection of a non-default InfiniBand
partition key.
---
ibv-conduit/README | 6 +++++
ibv-conduit/gasnet_core.c | 39 ++++++++++++++++++++++++++++++
ibv-conduit/gasnet_core_connect.c | 4 +--
ibv-conduit/gasnet_core_internal.h | 1 +
4 files changed, 48 insertions(+), 2 deletions(-)
diff --git a/ibv-conduit/README b/ibv-conduit/README
index 5c48b72b4..2d4f7be96 100644
--- a/ibv-conduit/README
+++ b/ibv-conduit/README
@@ -236,6 +236,12 @@ Paul H. Hargrove <PHHargrove@lbl.gov>
the available HCAs and the status of their ports.
The default is no filter.
+ + GASNET_IBV_PKEY
+ If set, this specifies the 15-bit InfiniBand Partition Key to use.
+ Valid values are in the range 2 to 0x7fff.
+ For compatibility, the membership bit (0x8000) is ignored.
+ The default is to use the Partition Key installed at table index 0.
+
+ GASNET_QP_TIMEOUT
This sets the timeout value used to configure InfiniBand QueuePairs.
The IB specification uses (4.096us * 2^qp_timeout) as the length of
diff --git a/ibv-conduit/gasnet_core.c b/ibv-conduit/gasnet_core.c
index 750b1c94a..ad3541f3f 100644
--- a/ibv-conduit/gasnet_core.c
+++ b/ibv-conduit/gasnet_core.c
@@ -17,6 +17,10 @@
#include <sys/time.h>
#include <sys/resource.h>
+// ntohs() should be in one of these:
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
GASNETI_IDENT(gasnetc_IdentString_Version, "$GASNetCoreLibraryVersion: " GASNET_CORE_VERSION_STR " $");
GASNETI_IDENT(gasnetc_IdentString_Name, "$GASNetCoreLibraryName: " GASNET_CORE_NAME_STR " $");
@@ -1470,6 +1474,17 @@ static void gasnetc_probe_ports(int max_ports) {
"'gasnet/ibv-conduit/README'.\n", num_hcas, current, enable, num_hcas);
}
+ int64_t pkey = gasnett_getenv_int_withdefault("GASNET_IBV_PKEY", -1, 0);
+ uint64_t pkey_mask = ~(uint64_t)0x8000; // to strip membership bit
+ if (pkey == -1) {
+ // Nothing to do
+ } else {
+ pkey &= pkey_mask;
+ if ((pkey > 0x7fff) || (pkey < 2)) {
+ gasneti_fatalerror("Invalid GASNET_IBV_PKEY '%s'", gasnett_getenv("GASNET_IBV_PKEY"));
+ }
+ }
+
/* Loop over list of HCAs */
for (curr_hca = 0;
(hca_count < GASNETC_IB_MAX_HCAS) && (port_count < max_ports) && (curr_hca < num_hcas);
@@ -1526,6 +1541,30 @@ static void gasnetc_probe_ports(int max_ports) {
++found;
this_port->port_num = curr_port;
this_port->hca_index = hca_count;
+ if (pkey < 0) {
+ GASNETI_TRACE_PRINTF(C,("Using default pkey_index=0 for HCA '%s', port %d",
+ hca_name, curr_port));
+ this_port->pkey_index = 0;
+ } else {
+ int i;
+ for (i = 0; i < hca_cap.max_pkeys; ++i) {
+ uint16_t pkey_val;
+ if (ibv_query_pkey(hca_handle, curr_port, i, &pkey_val)) {
+ gasneti_fatalerror("Failed to query pkeys for HCA '%s', port %d", hca_name, curr_port);
+ }
+ pkey_val = ntohs(pkey_val) & pkey_mask;
+ if (pkey_val == pkey) {
+ GASNETI_TRACE_PRINTF(C,("Using pkey_index %d for HCA '%s', port %d",
+ i, hca_name, curr_port));
+ this_port->pkey_index = i;
+ break;
+ }
+ }
+ if (i == hca_cap.max_pkeys) {
+ gasneti_fatalerror("Failed to locate index of requested pkey 0x%04x for HCA '%s', port %d",
+ (unsigned int)pkey, hca_name, curr_port);
+ }
+ }
if (gasnetc_qp_rd_atom) { /* Zero means use HCA/port limit */
int limit = MIN(hca_cap.max_qp_init_rd_atom, hca_cap.max_qp_rd_atom);
if (gasnetc_qp_rd_atom > limit) {
diff --git a/ibv-conduit/gasnet_core_connect.c b/ibv-conduit/gasnet_core_connect.c
index 65b4a4477..647f3806a 100644
--- a/ibv-conduit/gasnet_core_connect.c
+++ b/ibv-conduit/gasnet_core_connect.c
@@ -632,7 +632,6 @@ gasnetc_qp_reset2init(gasnetc_conn_info_t *conn_info)
qp_mask = (enum ibv_qp_attr_mask)(IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
qp_attr.qp_state = IBV_QPS_INIT;
- qp_attr.pkey_index = 0;
qp_attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
GASNETC_FOR_EACH_QPI(conn_info, qpi, cep) {
@@ -644,6 +643,7 @@ gasnetc_qp_reset2init(gasnetc_conn_info_t *conn_info)
: IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
#endif
qp_attr.port_num = port->port_num;
+ qp_attr.pkey_index = port->pkey_index;
#if GASNETC_IBV_XRC
if (gasnetc_use_xrc) {
@@ -1308,7 +1308,7 @@ gasnetc_qp_setup_ud(gasnetc_port_info_t *port, int fully_connected)
/* RESET -> INIT */
qp_mask = (enum ibv_qp_attr_mask)(IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_QKEY);
qp_attr.qp_state = IBV_QPS_INIT;
- qp_attr.pkey_index = 0;
+ qp_attr.pkey_index = port->pkey_index;
qp_attr.qkey = my_qkey;
qp_attr.port_num = port->port_num;
rc = ibv_modify_qp(conn_ud_qp, &qp_attr, qp_mask);
diff --git a/ibv-conduit/gasnet_core_internal.h b/ibv-conduit/gasnet_core_internal.h
index d373ffd9a..eb9b8836f 100644
--- a/ibv-conduit/gasnet_core_internal.h
+++ b/ibv-conduit/gasnet_core_internal.h
@@ -599,6 +599,7 @@ struct gasnetc_cep_t_ {
typedef struct {
int hca_index; /* Slot in gasnetc_hca[] */
uint8_t port_num; /* Port number */
+ uint16_t pkey_index;
struct ibv_port_attr port; /* Port info */
int rd_atom;
uint16_t *remote_lids;
--
2.48.1