Commits

Nathan Hjelm committed 013bb5f

hide btl segment keys

  • Participants
  • Parent commits 9b592b4

Comments (0)

Files changed (49)

ompi/mca/btl/btl.h

     /** Address of the memory */
     ompi_ptr_t seg_addr;        
      /** Length in bytes */
-    uint32_t   seg_len;           
-#if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
-    /** Heterogeneous padding */
-    uint8_t    seg_padding[4];     
-#endif
-    /** Memory segment key required by some RDMA networks */
-    union {
-        uint32_t  key32[4];
-        uint64_t  key64[2];
-        uint8_t   key8[16];
-#if OMPI_CUDA_SUPPORT
-        uint8_t cudakey[128]; /* 64 bytes for CUDA mem handle, 64 bytes for CUDA event handle */
-#endif /* OMPI_CUDA_SUPPORT */
-    } seg_key;
-#if OMPI_CUDA_SUPPORT
-    /** Address of the entire memory handle */
-    ompi_ptr_t memh_seg_addr;        
-     /** Length in bytes of entire memory handle */
-    uint32_t   memh_seg_len;           
-#endif /* OMPI_CUDA_SUPPORT */
+    uint64_t   seg_len;
 };
 typedef struct mca_btl_base_segment_t mca_btl_base_segment_t;
 
  */
 #define MCA_BTL_DES_MAX_SEGMENTS 16
 
+/**
+ * Maximum size of a btl segment (NTH: does it really save us anything
+ * to hardcode this?)
+ */
+#define MCA_BTL_SEG_MAX_SIZE 256
+
 /* 
  *  BTL base header, stores the tag at a minimum 
  */ 
     uint32_t    btl_latency;          /**< relative ranking of latency used to prioritize btls */
     uint32_t    btl_bandwidth;        /**< bandwidth (Mbytes/sec) supported by each endpoint */
     uint32_t    btl_flags;            /**< flags (put/get...) */
+    size_t      btl_seg_size;         /**< size of a btl segment */
 
     /* BTL function table */
     mca_btl_base_module_add_procs_fn_t      btl_add_procs;

ompi/mca/btl/mx/btl_mx.c

     if( OPAL_UNLIKELY(NULL == frag) ) {
         return NULL;
     }
-    frag->segment[0].seg_len = 
+    frag->segment[0].base.seg_len = 
         size <= mx_btl->super.btl_eager_limit ? 
         size : mx_btl->super.btl_eager_limit ;
-    frag->segment[0].seg_addr.pval = (void*)(frag+1);
+    frag->segment[0].base.seg_addr.pval = (void*)(frag+1);
     frag->base.des_src = frag->segment;
     frag->base.des_src_cnt = 1;
     frag->base.des_flags = flags;
             return NULL;
         }
         frag->base.des_src_cnt = 1;
-        iov.iov_base = (void*)((unsigned char*)frag->segment[0].seg_addr.pval + reserve);
+        iov.iov_base = (void*)((unsigned char*)frag->segment[0].base.seg_addr.pval + reserve);
     }
 
     iov.iov_len = max_data;
     *size = max_data;
 
     if( 1 == frag->base.des_src_cnt ) {
-        frag->segment[0].seg_len = reserve + max_data;
+        frag->segment[0].base.seg_len = reserve + max_data;
         if( 0 == reserve )
-            frag->segment[0].seg_addr.pval = iov.iov_base;
+            frag->segment[0].base.seg_addr.pval = iov.iov_base;
     } else {
-        frag->segment[0].seg_len       = reserve;
-        frag->segment[1].seg_len       = max_data;
-        frag->segment[1].seg_addr.pval = iov.iov_base;
+        frag->segment[0].base.seg_len       = reserve;
+        frag->segment[1].base.seg_len       = max_data;
+        frag->segment[1].base.seg_addr.pval = iov.iov_base;
     }
     frag->base.des_src   = frag->segment;
     frag->base.des_flags = flags;
         return NULL;
     }
 
-    frag->segment[0].seg_len       = *size;
-    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment[0].seg_addr.pval) );
-    frag->segment[0].seg_key.key64[0] = (uint64_t)(intptr_t)frag;
+    frag->segment[0].base.seg_len       = *size;
+    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment[0].base.seg_addr.pval) );
+    frag->segment[0].key = (uint64_t)(intptr_t)frag;
 
-    mx_segment.segment_ptr    = frag->segment[0].seg_addr.pval;
-    mx_segment.segment_length = frag->segment[0].seg_len;
+    mx_segment.segment_ptr    = frag->segment[0].base.seg_addr.pval;
+    mx_segment.segment_length = frag->segment[0].base.seg_len;
 
     mx_return = mx_irecv( mx_btl->mx_endpoint, &mx_segment, 1,
-                          frag->segment[0].seg_key.key64[0], 
-                          BTL_MX_PUT_MASK, NULL, &(frag->mx_request) );
+			  frag->segment[0].key, BTL_MX_PUT_MASK,
+                          NULL, &(frag->mx_request) );
     if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
         opal_output( 0, "Fail to re-register a fragment with the MX NIC ...\n" );
         MCA_BTL_MX_FRAG_RETURN( btl, frag );
                            struct mca_btl_base_endpoint_t* endpoint,
                            struct mca_btl_base_descriptor_t* descriptor )
 {
+    mca_btl_mx_segment_t *src_seg = (mca_btl_mx_segment_t *) descriptor->des_src;
+    mca_btl_mx_segment_t *dst_seg = (mca_btl_mx_segment_t *) descriptor->des_dst;
     mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
     mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
     mx_segment_t mx_segment[2];
     descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
 
     do {
-        mx_segment[i].segment_ptr    = descriptor->des_src[i].seg_addr.pval;
-        mx_segment[i].segment_length = descriptor->des_src[i].seg_len;
+	mx_segment[i].segment_ptr    = src_seg[i].base.seg_addr.pval;
+        mx_segment[i].segment_length = src_seg[i].base.seg_len;
     } while (++i < descriptor->des_src_cnt);
 
     mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
-                          endpoint->mx_peer_addr,
-                          descriptor->des_dst[0].seg_key.key64[0], frag,
+                          endpoint->mx_peer_addr, dst_seg->key, frag,
                           &frag->mx_request );
     if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
         opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
                      mca_btl_base_tag_t tag )
    
 {
+    mca_btl_mx_segment_t *src_seg = (mca_btl_mx_segment_t *) descriptor->des_src;
+    mca_btl_mx_segment_t *dst_seg = (mca_btl_mx_segment_t *) descriptor->des_dst;
     mca_btl_mx_module_t* mx_btl = (mca_btl_mx_module_t*)btl;
     mca_btl_mx_frag_t* frag = (mca_btl_mx_frag_t*)descriptor;
     mx_segment_t mx_segment[2];
     frag->type      = MCA_BTL_MX_SEND;
 
     do {
-        mx_segment[i].segment_ptr    = descriptor->des_src[i].seg_addr.pval;
-        mx_segment[i].segment_length = descriptor->des_src[i].seg_len;
-        total_length += descriptor->des_src[i].seg_len;
+        mx_segment[i].segment_ptr    = src_seg[i].base.seg_addr.pval;
+        mx_segment[i].segment_length = src_seg[i].base.seg_len;
+        total_length += src_seg[i].base.seg_len;
     } while (++i < descriptor->des_src_cnt);
 
     tag64 = 0x01ULL | (((uint64_t)tag) << 8);
     mx_return = mx_isend( mx_btl->mx_endpoint, mx_segment, descriptor->des_src_cnt,
-                          endpoint->mx_peer_addr,
-                          tag64, frag, &frag->mx_request );
+                          endpoint->mx_peer_addr, tag64, frag, &frag->mx_request );
     if( OPAL_UNLIKELY(MX_SUCCESS != mx_return) ) {
         opal_output( 0, "mx_isend fails with error %s\n", mx_strerror(mx_return) );
         return OMPI_ERROR;

ompi/mca/btl/mx/btl_mx_component.c

                                          MCA_BTL_FLAGS_PUT |
                                          MCA_BTL_FLAGS_SEND |
                                          MCA_BTL_FLAGS_RDMA_MATCHED);
+    mca_btl_mx_module.super.btl_seg_size  = sizeof (mca_btl_mx_segment_t);
     mca_btl_mx_module.super.btl_bandwidth = 2000;
     mca_btl_mx_module.super.btl_latency = 5;
     mca_btl_base_param_register(&mca_btl_mx_component.super.btl_version,

ompi/mca/btl/mx/btl_mx_frag.h

 #define MCA_BTL_MX_RECV  0x02
 
 BEGIN_C_DECLS
+
+struct mca_btl_mx_segment_t {
+  mca_btl_base_segment_t base;
+  uint64_t key;
+};
     
 /**
  * MX send framxent derived type.
  */
 struct mca_btl_mx_frag_t {
     mca_btl_base_descriptor_t       base; 
-    mca_btl_base_segment_t          segment[2]; 
+    mca_btl_mx_segment_t            segment[2]; 
     struct mca_btl_base_endpoint_t* endpoint; 
     uint8_t                         type;
     mx_request_t                    mx_request;

ompi/mca/btl/openib/btl_openib.c

         return NULL;
 
     /* not all upper layer users set this */
-    to_base_frag(item)->segment.seg_len = size;
+    to_base_frag(item)->segment.base.seg_len = size;
     to_base_frag(item)->base.order = order;
     to_base_frag(item)->base.des_flags = flags;
 
         }
 
         total_length = size + frag->coalesced_length +
-            to_base_frag(frag)->segment.seg_len +
+            to_base_frag(frag)->segment.base.seg_len +
             sizeof(mca_btl_openib_header_coalesced_t);
 
         qp = to_base_frag(frag)->base.order;
         sfrag->hdr->tag = MCA_BTL_TAG_BTL;
         ctrl_hdr->type = MCA_BTL_OPENIB_CONTROL_COALESCED;
         clsc_hdr->tag = org_tag;
-        clsc_hdr->size = to_base_frag(sfrag)->segment.seg_len;
-        clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.seg_len;
+        clsc_hdr->size = to_base_frag(sfrag)->segment.base.seg_len;
+        clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.base.seg_len;
         if(ep->nbo)
             BTL_OPENIB_HEADER_COALESCED_HTON(*clsc_hdr);
         sfrag->coalesced_length = sizeof(mca_btl_openib_control_header_t) +
 
     cfrag->hdr = (mca_btl_openib_header_coalesced_t*)((unsigned char*)(sfrag->hdr + 1) + 
                   sfrag->coalesced_length +
-                  to_base_frag(sfrag)->segment.seg_len);
+                  to_base_frag(sfrag)->segment.base.seg_len);
     cfrag->hdr = (mca_btl_openib_header_coalesced_t*)BTL_OPENIB_ALIGN_COALESCE_HDR(cfrag->hdr);
     cfrag->hdr->alloc_size = size;
 
     /* point coalesced frag pointer into a data buffer */
-    to_base_frag(cfrag)->segment.seg_addr.pval = cfrag->hdr + 1;
-    to_base_frag(cfrag)->segment.seg_len = size;
+    to_base_frag(cfrag)->segment.base.seg_addr.pval = cfrag->hdr + 1;
+    to_base_frag(cfrag)->segment.base.seg_len = size;
 
     /* save coalesced fragment on a main fragment; we will need it after send
      * completion to free it and to call upper layer callback */
 
             to_base_frag(frag)->base.order = order;
             to_base_frag(frag)->base.des_flags = flags;
-            to_base_frag(frag)->segment.seg_len = max_data;
-            to_base_frag(frag)->segment.seg_addr.pval = iov.iov_base;
-            to_base_frag(frag)->segment.seg_key.key32[0] =
-                (uint32_t)frag->sg_entry.lkey;
+            to_base_frag(frag)->segment.base.seg_len = max_data;
+            to_base_frag(frag)->segment.base.seg_addr.pval = iov.iov_base;
+            to_base_frag(frag)->segment.key = frag->sg_entry.lkey;
 
             assert(MCA_BTL_NO_ORDER == order);
 
-            BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64
-                         " frag->segment.seg_key.key32[0] = %" PRIu32,
-                         frag->sg_entry.lkey,
-                         frag->sg_entry.addr,
-                         frag->sg_entry.lkey));
+            BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64,
+                         frag->sg_entry.lkey, frag->sg_entry.addr));
 
             return &to_base_frag(frag)->base;
         }
         return NULL;
 
     iov.iov_len = max_data;
-    iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*)to_base_frag(frag)->segment.seg_addr.pval +
+    iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*)to_base_frag(frag)->segment.base.seg_addr.pval +
         reserve );
     rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
 
     *size = max_data;
 
     /* not all upper layer users set this */
-    to_base_frag(frag)->segment.seg_len = max_data + reserve;
+    to_base_frag(frag)->segment.base.seg_len = max_data + reserve;
 
     return &to_base_frag(frag)->base;
 }
     frag->sg_entry.lkey = openib_reg->mr->lkey;
     frag->sg_entry.addr = (uint64_t)(uintptr_t)buffer;
 
-    to_base_frag(frag)->segment.seg_addr.pval = buffer;
-    to_base_frag(frag)->segment.seg_len = *size;
-    to_base_frag(frag)->segment.seg_key.key32[0] = openib_reg->mr->rkey;
+    to_base_frag(frag)->segment.base.seg_addr.pval = buffer;
+    to_base_frag(frag)->segment.base.seg_len = *size;
+    to_base_frag(frag)->segment.key = openib_reg->mr->rkey;
     to_base_frag(frag)->base.order = order;
     to_base_frag(frag)->base.des_flags = flags;
 
     BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64 " "
-                 "frag->segment.seg_key.key32[0] = %" PRIu32,
-                 frag->sg_entry.lkey,
-                 frag->sg_entry.addr,
+                 "rkey = %" PRIu32, frag->sg_entry.lkey, frag->sg_entry.addr,
                  openib_reg->mr->rkey));
 
     return &to_base_frag(frag)->base;
     }
     frag = to_base_frag(item);
     hdr = to_send_frag(item)->hdr;
-    frag->segment.seg_len = size;
+    frag->segment.base.seg_len = size;
     frag->base.order = qp;
     frag->base.des_flags = flags;
     hdr->tag = tag;
     to_com_frag(item)->endpoint = ep;
 
     /* put match header */
-    memcpy(frag->segment.seg_addr.pval, header, header_size);
+    memcpy(frag->segment.base.seg_addr.pval, header, header_size);
 
     /* Pack data */
     if(payload_size) {
         struct iovec iov;
         uint32_t iov_count;
         /* pack the data into the supplied buffer */
-        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size);
+        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size);
         iov.iov_len  = max_data = payload_size;
         iov_count    = 1;
 
                     mca_btl_base_endpoint_t* ep,
                     mca_btl_base_descriptor_t* descriptor)
 {
+    mca_btl_openib_segment_t *src_seg = (mca_btl_openib_segment_t *) descriptor->des_src;
+    mca_btl_openib_segment_t *dst_seg = (mca_btl_openib_segment_t *) descriptor->des_dst;
     struct ibv_send_wr* bad_wr;
     mca_btl_openib_out_frag_t* frag = to_out_frag(descriptor);
     int qp = descriptor->order;
-    uint64_t rem_addr = descriptor->des_dst->seg_addr.lval;
-    uint32_t rkey = descriptor->des_dst->seg_key.key32[0];
+    uint64_t rem_addr = dst_seg->base.seg_addr.lval;
+    uint32_t rkey = dst_seg->key;
 
     assert(openib_frag_type(frag) == MCA_BTL_OPENIB_FRAG_SEND_USER ||
             openib_frag_type(frag) == MCA_BTL_OPENIB_FRAG_SEND);
     frag->sr_desc.wr.rdma.remote_addr = rem_addr;
     frag->sr_desc.wr.rdma.rkey = rkey;
 
-    to_com_frag(frag)->sg_entry.addr =
-        (uint64_t)(uintptr_t)descriptor->des_src->seg_addr.pval;
-    to_com_frag(frag)->sg_entry.length = descriptor->des_src->seg_len;
+    to_com_frag(frag)->sg_entry.addr = src_seg->base.seg_addr.lval;
+    to_com_frag(frag)->sg_entry.length = src_seg->base.seg_len;
     to_com_frag(frag)->endpoint = ep;
 #if HAVE_XRC
     if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp))
     /* Setting opcode on a frag constructor isn't enough since prepare_src
      * may return send_frag instead of put_frag */
     frag->sr_desc.opcode = IBV_WR_RDMA_WRITE;
-    frag->sr_desc.send_flags = ib_send_flags(descriptor->des_src->seg_len, &(ep->qps[qp]));
+    frag->sr_desc.send_flags = ib_send_flags(src_seg->seg_len, &(ep->qps[qp]));
     if(ibv_post_send(ep->qps[qp].qp->lcl_qp, &frag->sr_desc, &bad_wr))
         return OMPI_ERROR;
 
                     mca_btl_base_endpoint_t* ep,
                     mca_btl_base_descriptor_t* descriptor)
 {
+    mca_btl_openib_segment_t *src_seg = (mca_btl_openib_segment_t *) descriptor->des_src;
+    mca_btl_openib_segment_t *dst_seg = (mca_btl_openib_segment_t *) descriptor->des_dst;
     struct ibv_send_wr* bad_wr;
     mca_btl_openib_get_frag_t* frag = to_get_frag(descriptor);
     int qp = descriptor->order;
-    uint64_t rem_addr = descriptor->des_src->seg_addr.lval;
-    uint32_t rkey = descriptor->des_src->seg_key.key32[0];
+    uint64_t rem_addr = src_seg->base.seg_addr.lval;
+    uint32_t rkey = src_seg->key;
 
     assert(openib_frag_type(frag) == MCA_BTL_OPENIB_FRAG_RECV_USER);
 
     frag->sr_desc.wr.rdma.remote_addr = rem_addr;
     frag->sr_desc.wr.rdma.rkey = rkey;
 
-    to_com_frag(frag)->sg_entry.addr =
-        (uint64_t)(uintptr_t)descriptor->des_dst->seg_addr.pval;
-    to_com_frag(frag)->sg_entry.length  = descriptor->des_dst->seg_len;
+    to_com_frag(frag)->sg_entry.addr = dst_seg->base.seg_addr.lval;
+    to_com_frag(frag)->sg_entry.length = dst_seg->base.seg_len;
     to_com_frag(frag)->endpoint = ep;
 
 #if HAVE_XRC

ompi/mca/btl/openib/btl_openib_component.c

     mca_btl_openib_module_t *obtl = (mca_btl_openib_module_t*)btl;
     mca_btl_openib_endpoint_t* ep = to_com_frag(des)->endpoint;
     mca_btl_openib_control_header_t *ctl_hdr =
-        (mca_btl_openib_control_header_t *) to_base_frag(des)->segment.seg_addr.pval;
+        (mca_btl_openib_control_header_t *) to_base_frag(des)->segment.base.seg_addr.pval;
     mca_btl_openib_eager_rdma_header_t *rdma_hdr;
     mca_btl_openib_header_coalesced_t *clsc_hdr =
         (mca_btl_openib_header_coalesced_t*)(ctl_hdr + 1);
             mca_btl_base_active_message_trigger[MCA_BTL_TAG_IB].cbfunc = btl_openib_control;
             mca_btl_base_active_message_trigger[MCA_BTL_TAG_IB].cbdata = NULL;
 
+	    openib_btl->super.btl_seg_size = sizeof (mca_btl_openib_segment_t);
+
             /* Check bandwidth configured for this device */
             sprintf(param, "bandwidth_%s", ibv_get_device_name(device->ib_dev));
             openib_btl->super.btl_bandwidth =
 static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag)
 {
     mca_btl_openib_control_header_t* chdr =
-        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval;
+        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval;
     return (MCA_BTL_TAG_BTL == frag->hdr->tag) &&
         (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type);
 }
 static bool inline is_cts_message(const mca_btl_openib_recv_frag_t *frag)
 {
     mca_btl_openib_control_header_t* chdr =
-        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval;
+        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval;
     return (MCA_BTL_TAG_BTL == frag->hdr->tag) &&
         (MCA_BTL_OPENIB_CONTROL_CTS == chdr->type);
 }
             OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
             frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) -
                 size - BTL_OPENIB_FTR_PADDING(size) + sizeof(mca_btl_openib_footer_t));
-            to_base_frag(frag)->segment.seg_addr.pval =
+            to_base_frag(frag)->segment.base.seg_addr.pval =
                 ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
 
             ret = btl_openib_handle_incoming(btl, to_com_frag(frag)->endpoint,

ompi/mca/btl/openib/btl_openib_endpoint.c

     base_des->des_cbdata = NULL;
     base_des->des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     base_des->order = mca_btl_openib_component.credits_qp;
-    openib_frag->segment.seg_len = sizeof(mca_btl_openib_control_header_t);
+    openib_frag->segment.base.seg_len = sizeof(mca_btl_openib_control_header_t);
     com_frag->endpoint = endpoint;
 
     sc_frag->hdr->tag = MCA_BTL_TAG_BTL;
     sc_frag->hdr->credits = 0;
 
     ctl_hdr = (mca_btl_openib_control_header_t*)
-        openib_frag->segment.seg_addr.pval;
+        openib_frag->segment.base.seg_addr.pval;
     ctl_hdr->type = MCA_BTL_OPENIB_CONTROL_CTS;
 
     /* Send the fragment */
         to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;;
         to_com_frag(frag)->endpoint = endpoint;
         frag->hdr->tag = MCA_BTL_TAG_BTL;
-        to_base_frag(frag)->segment.seg_len =
+        to_base_frag(frag)->segment.base.seg_len =
             sizeof(mca_btl_openib_rdma_credits_header_t);
     }
 
     assert(frag->qp_idx == qp);
     credits_hdr = (mca_btl_openib_rdma_credits_header_t*)
-        to_base_frag(frag)->segment.seg_addr.pval;
+        to_base_frag(frag)->segment.base.seg_addr.pval;
     if(OMPI_SUCCESS == acquire_eager_rdma_send_credit(endpoint)) {
         do_rdma = true;
     } else {
     to_base_frag(frag)->base.des_cbdata = NULL;
     to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
-    to_base_frag(frag)->segment.seg_len =
+    to_base_frag(frag)->segment.base.seg_len =
         sizeof(mca_btl_openib_eager_rdma_header_t);
     to_com_frag(frag)->endpoint = endpoint;
 
     frag->hdr->tag = MCA_BTL_TAG_BTL;
-    rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
+    rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
     rdma_hdr->control.type = MCA_BTL_OPENIB_CONTROL_RDMA;
     rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey;
     rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval);
         to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_EAGER_RDMA;
         to_com_frag(frag)->endpoint = endpoint;
         frag->ftr = (mca_btl_openib_footer_t*)
-            ((char*)to_base_frag(frag)->segment.seg_addr.pval +
+            ((char*)to_base_frag(frag)->segment.base.seg_addr.pval +
              mca_btl_openib_component.eager_limit);
 
         MCA_BTL_OPENIB_RDMA_MAKE_REMOTE(frag->ftr);

ompi/mca/btl/openib/btl_openib_failover.c

     to_base_frag(frag)->base.des_cbdata = NULL;
     to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
-    to_base_frag(frag)->segment.seg_len =
+    to_base_frag(frag)->segment.base.seg_len =
         sizeof(mca_btl_openib_broken_connection_header_t);
     to_com_frag(frag)->endpoint = newep;
 
     frag->hdr->tag = MCA_BTL_TAG_BTL;
-    bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
+    bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
     bc_hdr->control.type = type;
     bc_hdr->lid = endpoint->endpoint_btl->port_info.lid;
     bc_hdr->subnet_id = endpoint->endpoint_btl->port_info.subnet_id;
 
         frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) -
                size + sizeof(mca_btl_openib_footer_t));
-        to_base_frag(frag)->segment.seg_addr.pval =
+        to_base_frag(frag)->segment.base.seg_addr.pval =
                ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
 
-        chdr = to_base_frag(frag)->segment.seg_addr.pval;
+        chdr = to_base_frag(frag)->segment.base.seg_addr.pval;
         if ((MCA_BTL_TAG_BTL == frag->hdr->tag) &&
             (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type)) {
             opal_output(0, "tag[%d] is credit message", i);

ompi/mca/btl/openib/btl_openib_frag.c

 
     if(reg) {
         frag->sg_entry.lkey = reg->mr->lkey;
-        base_frag->segment.seg_key.key32[0] = reg->mr->lkey;
+        base_frag->segment.key = reg->mr->lkey;
     }
 }
 
         (((unsigned char*)base_frag->base.super.ptr) +
         sizeof(mca_btl_openib_header_coalesced_t) +
         sizeof(mca_btl_openib_control_header_t));
-    base_frag->segment.seg_addr.pval = frag->hdr + 1;
+    base_frag->segment.base.seg_addr.pval = frag->hdr + 1;
     to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
     frag->coalesced_length = 0;
     OBJ_CONSTRUCT(&frag->coalesced_frags, opal_list_t);
     base_frag->type = MCA_BTL_OPENIB_FRAG_RECV;
 
     frag->hdr = (mca_btl_openib_header_t*)base_frag->base.super.ptr;
-    base_frag->segment.seg_addr.pval =
+    base_frag->segment.base.seg_addr.pval =
         ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
     to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
 
     to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_CONTROL;
     /* adjusting headers because there is no coalesce header in control messages */
     frag->hdr = frag->chdr;
-    to_base_frag(frag)->segment.seg_addr.pval = frag->hdr + 1;
+    to_base_frag(frag)->segment.base.seg_addr.pval = frag->hdr + 1;
     to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
 }
 

ompi/mca/btl/openib/btl_openib_frag.h

  * IB fragment derived type.
  */
 
+typedef struct mca_btl_openib_segment_t {
+    mca_btl_base_segment_t base;
+    uint32_t key;
+} mca_btl_openib_segment_t;
+
 /* base openib frag */
 typedef struct mca_btl_openib_frag_t {
     mca_btl_base_descriptor_t base;
-    mca_btl_base_segment_t segment;
+    mca_btl_openib_segment_t segment;
     mca_btl_openib_frag_type_t type;
     ompi_free_list_t* list;
 } mca_btl_openib_frag_t;

ompi/mca/btl/portals/btl_portals.c

     if (size <= mca_btl_portals_module.super.btl_eager_limit) { 
         OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, rc); 
         if (OMPI_SUCCESS != rc) return NULL;
-        frag->segments[0].seg_len = size;
+        frag->segments[0].base.seg_len = size;
     } else { 
         OMPI_BTL_PORTALS_FRAG_ALLOC_MAX(&mca_btl_portals_module, frag, rc); 
         if (OMPI_SUCCESS != rc) return NULL;
-        frag->segments[0].seg_len = 
+        frag->segments[0].base.seg_len = 
             size <= mca_btl_portals_module.super.btl_max_send_size ? 
             size : mca_btl_portals_module.super.btl_max_send_size ; 
     }
         }
         
         iov.iov_len = max_data;
-        iov.iov_base = (unsigned char*) frag->segments[0].seg_addr.pval + reserve;
+        iov.iov_base = (unsigned char*) frag->segments[0].base.seg_addr.pval + reserve;
         ret = opal_convertor_pack(convertor, &iov, &iov_count, 
                                   &max_data );
         *size  = max_data;
             return NULL;
         }
 
-        frag->segments[0].seg_len = max_data + reserve;
+        frag->segments[0].base.seg_len = max_data + reserve;
         frag->base.des_src_cnt = 1;
 
     } else {
 
         opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
 
-        frag->segments[0].seg_len = max_data;
-        frag->segments[0].seg_addr.pval = iov.iov_base;
-        frag->segments[0].seg_key.key64[0] = 
-            OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
+        frag->segments[0].base.seg_len = max_data;
+        frag->segments[0].base.seg_addr.pval = iov.iov_base;
+        frag->segments[0].key = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
         frag->base.des_src_cnt = 1;
 
         /* either a put or get.  figure out which later */
                              "rdma src posted for frag 0x%lx, callback 0x%lx, bits %"PRIu64", flags say %d" ,
                              (unsigned long) frag, 
                              (unsigned long) frag->base.des_cbfunc,
-                             frag->segments[0].seg_key.key64[0], flags));
+                             frag->segments[0].key, flags));
 
         /* create a match entry */
         ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
                           OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                           *((mca_btl_base_endpoint_t*) peer),
-                          frag->segments[0].seg_key.key64[0], /* match */
+			  frag->segments[0].key, /* match */
                           0, /* ignore */
                           PTL_UNLINK,
                           PTL_INS_AFTER,
         }
 
         /* setup the memory descriptor */
-        md.start = frag->segments[0].seg_addr.pval;
-        md.length = frag->segments[0].seg_len;
+        md.start = frag->segments[0].base.seg_addr.pval;
+        md.length = frag->segments[0].base.seg_len;
         md.threshold = PTL_MD_THRESH_INF;
         md.max_size = 0;
         md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
         return NULL;
     }
 
-    frag->segments[0].seg_len = *size;
-    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segments[0].seg_addr.pval) );
-    frag->segments[0].seg_key.key64[0] = 
-        OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
+    frag->segments[0].base.seg_len = *size;
+    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segments[0].base.seg_addr.pval) );
+    frag->segments[0].key = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
     frag->base.des_src = NULL;
     frag->base.des_src_cnt = 0;
     frag->base.des_dst = frag->segments;
                          "rdma dest posted for frag 0x%lx, callback 0x%lx, bits %" PRIu64 " flags %d",
                          (unsigned long) frag,
                          (unsigned long) frag->base.des_cbfunc,
-                         frag->segments[0].seg_key.key64[0],
+			 frag->segments[0].key;
                          flags));
 
     /* create a match entry */
     ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
                       OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                       *((mca_btl_base_endpoint_t*) peer),
-                      frag->segments[0].seg_key.key64[0], /* match */
+		      frag->segments[0].key, /* match */
                       0, /* ignore */
                       PTL_UNLINK,
                       PTL_INS_AFTER,
     }
 
     /* setup the memory descriptor. */
-    md.start = frag->segments[0].seg_addr.pval;
-    md.length = frag->segments[0].seg_len;
+    md.start = frag->segments[0].base.seg_addr.pval;
+    md.length = frag->segments[0].base.seg_len;
     md.threshold = PTL_MD_THRESH_INF;
     md.max_size = 0;
     md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;

ompi/mca/btl/portals/btl_portals_component.c

     mca_btl_portals_module.super.btl_flags = 
         MCA_BTL_FLAGS_RDMA |
         MCA_BTL_FLAGS_RDMA_MATCHED;
+    mca_btl_portals_module.super.btl_seg_size = sizeof (mca_btl_portals_segment_t);
     mca_btl_portals_module.super.btl_bandwidth = 1000;
     mca_btl_portals_module.super.btl_latency = 0;
 
     mca_btl_portals_frag_t *frag = NULL;
     mca_btl_portals_recv_block_t *block = NULL;
     mca_btl_base_tag_t tag;
+    mca_btl_base_segment_t seg[2];
 
     if (0 == mca_btl_portals_module.portals_num_procs) {
         return 0;
                                              ));
                         
                         OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,"received %d bytes \n", (int) ev.mlength));
-                        frag->segments[0].seg_addr.pval = &frag->data;
-                        frag->segments[0].seg_len = header_size;
+			frag->base.des_dst = seg;
+			seg[0].seg_addr.pval = &frag->data;
+			seg[0].seg_len = header_size;
                         if(ev.mlength) {
-                            frag->segments[1].seg_addr.pval = ((((char*) ev.md.start) + ev.offset));
-                            frag->segments[1].seg_len = ev.mlength;
+			    seg[1].seg_addr.pval = ((((char*) ev.md.start) + ev.offset));
+			    seg[1].seg_len = ev.mlength;
                             frag->base.des_dst_cnt = 2;
                         } else { 
                             frag->base.des_dst_cnt = 1;

ompi/mca/btl/portals/btl_portals_frag.h

 
 BEGIN_C_DECLS
 
+struct mca_btl_portals_segment_t {
+  mca_btl_base_segment_t base;
+  ptl_match_bits_t key;
+};
+typedef struct mca_btl_portals_segment_t mca_btl_portals_segment_t;
+
 /**
  * Portals send fragment derived type
  */
 struct mca_btl_portals_frag_t {
     mca_btl_base_descriptor_t base; 
-    mca_btl_base_segment_t segments[1]; 
+    mca_btl_portals_segment_t segments[1]; 
     /* needed for retransmit case */
     struct mca_btl_base_endpoint_t *endpoint; 
     /* needed for retransmit case */

ompi/mca/btl/self/btl_self.c

     frag->base.des_flags = flags;
     frag->base.des_src          = &frag->segment;
     frag->base.des_src_cnt      = 1;
-    frag->segment.seg_key.key64[0] = (uint64_t)(intptr_t)convertor;
+
     return &frag->base;
 }
 
     /* setup descriptor to point directly to user buffer */
     opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
     frag->segment.seg_len = reserve + max_data;
-    frag->segment.seg_key.key64[0] = (uint64_t)(intptr_t)convertor;
     frag->base.des_dst = &frag->segment;
     frag->base.des_dst_cnt = 1;
     frag->base.des_flags = flags;

ompi/mca/btl/self/btl_self_component.c

     mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;
     mca_btl_self.btl_min_rdma_pipeline_size = 0;
     mca_btl_self.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE;
+    mca_btl_self.btl_seg_len = sizeof (mca_btl_base_segment_t);
     mca_btl_self.btl_bandwidth = 100;
     mca_btl_self.btl_latency = 0;
     mca_btl_base_param_register(&mca_btl_self_component.super.btl_version,

ompi/mca/btl/sm/btl_sm.c

     }
 
     if (OPAL_LIKELY(frag != NULL)) {
-        frag->segment.seg_len = size;
+        frag->segment.base.seg_len = size;
         frag->base.des_flags = flags;
     }
     return (mca_btl_base_descriptor_t*)frag;
         }
         iov.iov_len = max_data;
         iov.iov_base =
-            (IOVBASE_TYPE*)(((unsigned char*)(frag->segment.seg_addr.pval)) +
-                            reserve);
+	    (IOVBASE_TYPE*)(frag->segment.base.seg_addr.lval + reserve);
 
         rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
         if( OPAL_UNLIKELY(rc < 0) ) {
             MCA_BTL_SM_FRAG_RETURN(frag);
             return NULL;
         }
-        frag->segment.seg_len = reserve + max_data;
+        frag->segment.base.seg_len = reserve + max_data;
 #if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
     } else {
 #if OMPI_BTL_SM_HAVE_KNEM
             MCA_BTL_SM_FRAG_RETURN(frag);
             return NULL;
         }
-        frag->segment.seg_addr.pval = iov.iov_base;
-        frag->segment.seg_len = max_data;
+        frag->segment.base.seg_addr.pval = iov.iov_base;
+        frag->segment.base.seg_len = max_data;
 
 #if OMPI_BTL_SM_HAVE_KNEM
         if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
             if (OPAL_UNLIKELY(ioctl(sm_btl->knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
                 return NULL;
             }
-            frag->segment.seg_key.key64[0] = knem_cr.cookie;
+            frag->segment.key = knem_cr.cookie;
         }
 #endif /* OMPI_BTL_SM_HAVE_KNEM */
 
 #if OMPI_BTL_SM_HAVE_CMA
         if (OPAL_LIKELY(mca_btl_sm_component.use_cma)) {
             /* Encode the pid as the key */
-            frag->segment.seg_key.key64[0] = getpid();
+            frag->segment.key = getpid();
         }
 #endif /* OMPI_BTL_SM_HAVE_CMA */
     }
 #if 0
 #define MCA_BTL_SM_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)          \
     do {                                                                \
-        char* _memory = (char*)(sm_frag)->segment.seg_addr.pval +       \
-            (sm_frag)->segment.seg_len;                                 \
+	char* _memory = (char*)(sm_frag)->segment.base.seg_addr.pval +	\
+            (sm_frag)->segment.base.seg_len;				\
         int* _intmem;                                                   \
         size_t align = (intptr_t)_memory & 0xFUL;                       \
         switch( align & 0x3 ) {                                         \
         }
 
         /* fill in fragment fields */
-        frag->segment.seg_len = length;
+        frag->segment.base.seg_len = length;
         frag->hdr->len        = length;
         assert( 0 == (flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) );
         frag->base.des_flags = flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;   /* why do any flags matter here other than OWNERSHIP? */
         frag->endpoint = endpoint;
 
         /* write the match header (with MPI comm/tag/etc. info) */
-        memcpy( frag->segment.seg_addr.pval, header, header_size );
+        memcpy( frag->segment.base.seg_addr.pval, header, header_size );
 
         /* write the message data if there is any */
         /*
             struct iovec iov;
             uint32_t iov_count;
             /* pack the data into the supplied buffer */
-            iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size);
+            iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size);
             iov.iov_len  = max_data = payload_size;
             iov_count    = 1;
 
     }
 
     /* available header space */
-    frag->hdr->len = frag->segment.seg_len;
+    frag->hdr->len = frag->segment.base.seg_len;
     /* type of message, pt-2-pt, one-sided, etc */
     frag->hdr->tag = tag;
 
         return NULL;
     }
 
-    frag->segment.seg_len = *size;
-    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
+    frag->segment.base.seg_len = *size;
+    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.base.seg_addr.pval) );
     
     frag->base.des_src = NULL;
     frag->base.des_src_cnt = 0;
     int btl_ownership;
     mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
     mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
-    mca_btl_base_segment_t *src = des->des_src;
-    mca_btl_base_segment_t *dst = des->des_dst;
+    mca_btl_sm_segment_t *src = des->des_src;
+    mca_btl_sm_segment_t *dst = des->des_dst;
 #if OMPI_BTL_SM_HAVE_KNEM
     if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
         struct knem_cmd_inline_copy icopy;
     
         /* Fill in the ioctl data fields.  There's no async completion, so
            we don't need to worry about getting a slot, etc. */
-        recv_iovec.base = (uintptr_t) dst->seg_addr.pval;
-        recv_iovec.len =  dst->seg_len;
+        recv_iovec.base = (uintptr_t) dst->base.seg_addr.pval;
+        recv_iovec.len =  dst->base.seg_len;
         icopy.local_iovec_array = (uintptr_t)&recv_iovec;
         icopy.local_iovec_nr = 1;
-        icopy.remote_cookie = src->seg_key.key64[0];
+        icopy.remote_cookie = src->key;
         icopy.remote_offset = 0;
         icopy.write = 0;
 
            value is 0 (i.e., the MCA param was set to 0), the segment size
            will never be larger than it, so DMA will never be used. */
         icopy.flags = 0;
-        if (mca_btl_sm_component.knem_dma_min <= dst->seg_len) {
+        if (mca_btl_sm_component.knem_dma_min <= dst->base.seg_len) {
             icopy.flags = mca_btl_sm_component.knem_dma_flag;
         }
         /* synchronous flags only, no need to specify icopy.async_status_index */
         pid_t remote_pid;
         int val;
 
-        remote_address = (char *) src->seg_addr.pval;
-        remote_length = src->seg_len;
+        remote_address = (char *) src->base.seg_addr.pval;
+        remote_length = src->base.seg_len;
 
-        local_address = (char *) dst->seg_addr.pval;
-        local_length = dst->seg_len;
+        local_address = (char *) dst->base,seg_addr.pval;
+        local_length = dst->base.seg_len;
 
-        remote_pid = src->seg_key.key64[0];
-        remote.iov_base = src->seg_addr.pval;
-        remote.iov_len = src->seg_len;
-        local.iov_base = dst->seg_addr.pval;
-        local.iov_len = dst->seg_len;
+        remote_pid = src->key;
+        remote.iov_base = src->base.seg_addr.pval;
+        remote.iov_len = src->base.seg_len;
+        local.iov_base = dst->base.seg_addr.pval;
+        local.iov_len = dst->base.seg_len;
 
         val = process_vm_readv(remote_pid, &local, 1, &remote, 1, 0);
 
     int btl_ownership;
     mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
     mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
-    mca_btl_base_segment_t *src = des->des_src;
-    mca_btl_base_segment_t *dst = des->des_dst;
+    mca_btl_sm_segment_t *src = des->des_src;
+    mca_btl_sm_segment_t *dst = des->des_dst;
     struct knem_cmd_inline_copy icopy;
     struct knem_cmd_param_iovec recv_iovec;
     
 
     /* We have a slot, so fill in the data fields.  Bump the
        first_avail and num_used counters. */
-    recv_iovec.base = (uintptr_t) dst->seg_addr.pval;
-    recv_iovec.len =  dst->seg_len;
+    recv_iovec.base = (uintptr_t) dst->base.seg_addr.pval;
+    recv_iovec.len =  dst->base.seg_len;
     icopy.local_iovec_array = (uintptr_t)&recv_iovec;
     icopy.local_iovec_nr = 1;
     icopy.write = 0;
         sm_btl->knem_status_first_avail = 0;
     }
     ++sm_btl->knem_status_num_used;
-    icopy.remote_cookie = src->seg_key.key64[0];
+    icopy.remote_cookie = src->key;
     icopy.remote_offset = 0;
 
     /* Use the DMA flag if knem supports it *and* the segment length
        is greater than the cutoff */
     icopy.flags = KNEM_FLAG_ASYNCDMACOMPLETE;
-    if (mca_btl_sm_component.knem_dma_min <= dst->seg_len) {
+    if (mca_btl_sm_component.knem_dma_min <= dst->base.seg_len) {
         icopy.flags = mca_btl_sm_component.knem_dma_flag;
     }
 

ompi/mca/btl/sm/btl_sm_component.c

     }
 
 #endif
+    mca_btl_sm.super.seg_len = sizeof (mca_btl_sm_segment_t);
     mca_btl_sm.super.btl_bandwidth = 9000;  /* Mbs */
     mca_btl_sm.super.btl_latency   = 1;     /* Microsecs */
 

ompi/mca/btl/sm/btl_sm_frag.c

     if(frag->hdr != NULL) {
         frag->hdr->frag = (mca_btl_sm_frag_t*)((uintptr_t)frag |
             MCA_BTL_SM_FRAG_ACK);
-        frag->segment.seg_addr.pval = ((char*)frag->hdr) +
+        frag->segment.base.seg_addr.pval = ((char*)frag->hdr) +
             sizeof(mca_btl_sm_hdr_t);
         frag->hdr->my_smp_rank = mca_btl_sm_component.my_smp_rank;
     }
-    frag->segment.seg_len = frag->size;
+    frag->segment.base.seg_len = frag->size;
     frag->base.des_src = &frag->segment;
     frag->base.des_src_cnt = 1;
     frag->base.des_dst = &frag->segment;

ompi/mca/btl/sm/btl_sm_frag.h

 };
 typedef struct mca_btl_sm_hdr_t mca_btl_sm_hdr_t;
 
+struct mca_btl_sm_segment_t {
+    mca_btl_base_segment_t base;
+#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA 
+    uint64_t key;
+#endif
+};
+typedef struct mca_btl_sm_segment_t mca_btl_sm_segment_t;
+
 /**
  * shared memory send fragment derived type.
  */
 struct mca_btl_sm_frag_t {
     mca_btl_base_descriptor_t base;
-    mca_btl_base_segment_t segment;
+    mca_btl_sm_segment_t segment;
     struct mca_btl_base_endpoint_t *endpoint;
     size_t size;
     /* pointer written to the FIFO, this is the base of the shared memory region */

ompi/mca/btl/smcuda/btl_smcuda.c

     }
 
     if (OPAL_LIKELY(frag != NULL)) {
-        frag->segment.seg_len = size;
+        frag->segment.base.seg_len = size;
         frag->base.des_flags = flags;
     }
     return (mca_btl_base_descriptor_t*)frag;
         }
         iov.iov_len = max_data;
         iov.iov_base =
-            (IOVBASE_TYPE*)(((unsigned char*)(frag->segment.seg_addr.pval)) +
-                            reserve);
+	    (IOVBASE_TYPE*)(frag->segment.base.seg_addr.lval + reserve);
 
         rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
         if( OPAL_UNLIKELY(rc < 0) ) {
             MCA_BTL_SMCUDA_FRAG_RETURN(frag);
             return NULL;
         }
-        frag->segment.seg_len = reserve + max_data;
+        frag->segment.base.seg_len = reserve + max_data;
 #if OMPI_CUDA_SUPPORT
     } else {
         /* Normally, we are here because we have a GPU buffer and we are preparing
            MCA_BTL_SMCUDA_FRAG_RETURN(frag);
             return NULL;
         }
-        frag->segment.seg_addr.pval = iov.iov_base;
-        frag->segment.seg_len = max_data;
-        memcpy(frag->segment.seg_key.cudakey, ((mca_mpool_common_cuda_reg_t *)registration)->memHandle,
+        frag->segment.base.seg_addr.pval = iov.iov_base;
+        frag->segment.base.seg_len = max_data;
+        memcpy(frag->segment.key, ((mca_mpool_common_cuda_reg_t *)registration)->memHandle,
                sizeof(((mca_mpool_common_cuda_reg_t *)registration)->memHandle) + 
                sizeof(((mca_mpool_common_cuda_reg_t *)registration)->evtHandle));
         frag->segment.memh_seg_addr.pval = registration->base;
 }
 
 #if 0
-#define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)          \
+#define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)	\
     do {                                                                \
-        char* _memory = (char*)(sm_frag)->segment.seg_addr.pval +       \
-            (sm_frag)->segment.seg_len;                                 \
+        char* _memory = (char*)(sm_frag)->segment.base.seg_addr.pval +	\
+            (sm_frag)->segment.base.seg_len;				\
         int* _intmem;                                                   \
         size_t align = (intptr_t)_memory & 0xFUL;                       \
         switch( align & 0x3 ) {                                         \
         }
 
         /* fill in fragment fields */
-        frag->segment.seg_len = length;
+        frag->segment.base.seg_len = length;
         frag->hdr->len        = length;
         assert( 0 == (flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) );
         frag->base.des_flags = flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;   /* why do any flags matter here other than OWNERSHIP? */
         frag->endpoint = endpoint;
 
         /* write the match header (with MPI comm/tag/etc. info) */
-        memcpy( frag->segment.seg_addr.pval, header, header_size );
+        memcpy( frag->segment.base.seg_addr.pval, header, header_size );
 
         /* write the message data if there is any */
         /*
             struct iovec iov;
             uint32_t iov_count;
             /* pack the data into the supplied buffer */
-            iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size);
+            iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size);
             iov.iov_len  = max_data = payload_size;
             iov_count    = 1;
 
     }
 
     /* available header space */
-    frag->hdr->len = frag->segment.seg_len;
+    frag->hdr->len = frag->segment.base.seg_len;
     /* type of message, pt-2-pt, one-sided, etc */
     frag->hdr->tag = tag;
 
         return NULL;
     }
     
-    frag->segment.seg_len = *size;
-    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
+    frag->segment.base.seg_len = *size;
+    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.base.seg_addr.pval) );
 
     frag->base.des_src = NULL;
     frag->base.des_src_cnt = 0;
                         struct mca_btl_base_endpoint_t* ep,
                         struct mca_btl_base_descriptor_t* descriptor)
 {
+    mca_btl_smcuda_segment_t *src_seg = (mca_btl_smcuda_segment_t *) descriptor->des_src;
+    mca_btl_smcuda_segment_t *dst_seg = (mca_btl_smcuda_segment_t *) descriptor->des_dst;
     mca_mpool_common_cuda_reg_t rget_reg;
     mca_mpool_common_cuda_reg_t *reg_ptr = &rget_reg;
     int btl_ownership;
      * garbage in the debugger.  */
     
     memset(&rget_reg, 0, sizeof(rget_reg));
-    memcpy(&rget_reg.memHandle, descriptor->des_src->seg_key.cudakey,
-           sizeof(descriptor->des_src->seg_key.cudakey));
+    memcpy(&rget_reg.memHandle, dst_seg->key, sizeof(dst_seg->key));
 
     /* Open the memory handle to the remote memory.  If it is cached, then
      * we just retrieve it from cache and avoid a call to open the handle.  That
      * remote memory which may lie somewhere in the middle. This is taken care of
      * a few lines down. Note that we hand in the peer rank just for debugging
      * support. */
-    rc = ep->mpool->mpool_register(ep->mpool, descriptor->des_src->memh_seg_addr.pval,
-                                   descriptor->des_src->memh_seg_len, ep->peer_smp_rank,
+    rc = ep->mpool->mpool_register(ep->mpool, src_seg->memh_seg_addr.pval,
+				   src_seg->memh_seg_len, ep->peer_smp_rank,
                                    (mca_mpool_base_registration_t **)&reg_ptr);
 
     if (OMPI_SUCCESS != rc) {
      * not equal the address that was used to retrieve the block.
      * Therefore, compute the offset and add it to the address of the
      * memory handle. */
-    offset = (unsigned char *)descriptor->des_src->seg_addr.pval - reg_ptr->base.base;
+    offset = src_seg->base.seg_addr.lval - reg_ptr->base.base;
     remote_memory_address = (unsigned char *)reg_ptr->base.alloc_base + offset;
     if (0 != offset) {
         opal_output(-1, "OFFSET=%d", (int)offset);
      * rget_reg, not reg_ptr, as we do not cache the event. */
     mca_common_wait_stream_synchronize(&rget_reg);
 
-    rc = mca_common_cuda_memcpy(descriptor->des_dst->seg_addr.pval, remote_memory_address,
-                                descriptor->des_dst->seg_len, "mca_btl_smcuda_get",
+    rc = mca_common_cuda_memcpy(dst_seg->base.seg_addr.pval, remote_memory_address,
+				dst_seg->base.seg_len, "mca_btl_smcuda_get",
                                 (mca_btl_base_descriptor_t *)frag, &done);
     if (OMPI_SUCCESS != rc) {
         /* Out of resources can be handled by upper layers. */

ompi/mca/btl/smcuda/btl_smcuda_component.c

 #if OMPI_CUDA_SUPPORT
     mca_btl_smcuda.super.btl_flags |= MCA_BTL_FLAGS_CUDA_GET;
 #endif /* OMPI_CUDA_SUPPORT */
+    mca_btl_smcuda.super.btl_seg_size = sizeof (mca_btl_sm_segment_t);
     mca_btl_smcuda.super.btl_bandwidth = 9000;  /* Mbs */
     mca_btl_smcuda.super.btl_latency   = 1;     /* Microsecs */
 
 int mca_btl_smcuda_component_progress(void)
 {
     /* local variables */
+    mca_btl_base_segment_t seg;
     mca_btl_smcuda_frag_t *frag;
     mca_btl_smcuda_frag_t Frag;
     sm_fifo_t *fifo = NULL;
 #endif
                 /* recv upcall */
                 reg = mca_btl_base_active_message_trigger + hdr->tag;
-                Frag.segment.seg_addr.pval = ((char*)hdr) +
-                    sizeof(mca_btl_smcuda_hdr_t);
-                Frag.segment.seg_len = hdr->len;
+		seg.seg_addr.pval = ((char*)hdr) + sizeof(mca_btl_smcuda_hdr_t);
+		seg.seg_len = hdr->len;
                 Frag.base.des_dst_cnt = 1;
-                Frag.base.des_dst = &(Frag.segment);
+                Frag.base.des_dst = &seg;
                 reg->cbfunc(&mca_btl_smcuda.super, hdr->tag, &(Frag.base),
                             reg->cbdata);
                 /* return the fragment */

ompi/mca/btl/smcuda/btl_smcuda_frag.c

     if(frag->hdr != NULL) {
         frag->hdr->frag = (mca_btl_smcuda_frag_t*)((uintptr_t)frag |
             MCA_BTL_SMCUDA_FRAG_ACK);
-        frag->segment.seg_addr.pval = ((char*)frag->hdr) +
+        frag->segment.base.seg_addr.pval = ((char*)frag->hdr) +
             sizeof(mca_btl_smcuda_hdr_t);
         frag->hdr->my_smp_rank = mca_btl_smcuda_component.my_smp_rank;
     }
-    frag->segment.seg_len = frag->size;
+    frag->segment.base.seg_len = frag->size;
     frag->base.des_src = &frag->segment;
     frag->base.des_src_cnt = 1;
     frag->base.des_dst = &frag->segment;

ompi/mca/btl/smcuda/btl_smcuda_frag.h

 };
 typedef struct mca_btl_smcuda_hdr_t mca_btl_smcuda_hdr_t;
 
+struct mca_btl_smcuda_segment_t {
+    mca_btl_base_segment_t base;
+#if OMPI_CUDA_SUPPORT
+    uint8_t key[128]; /* 64 bytes for CUDA mem handle, 64 bytes for CUDA event handle */
+    /** Address of the entire memory handle */
+    ompi_ptr_t memh_seg_addr;        
+     /** Length in bytes of entire memory handle */
+    uint32_t memh_seg_len;           
+#endif
+};
+typedef struct mca_btl_smcuda_segment_t mca_btl_smcuda_segment_t;
+
 /**
  * shared memory send fragment derived type.
  */
 struct mca_btl_smcuda_frag_t {
     mca_btl_base_descriptor_t base;
-    mca_btl_base_segment_t segment;
+    mca_btl_smcuda_segment_t segment;
     struct mca_btl_base_endpoint_t *endpoint;
 #if OMPI_CUDA_SUPPORT
     struct mca_mpool_base_registration_t *registration;

ompi/mca/btl/tcp/btl_tcp_component.c

                                        MCA_BTL_FLAGS_NEED_CSUM |
                                        MCA_BTL_FLAGS_NEED_ACK |
                                        MCA_BTL_FLAGS_HETEROGENEOUS_RDMA;
+    mca_btl_tcp_module.super.btl_seg_size = sizeof (mca_btl_base_segment_t);
     mca_btl_tcp_module.super.btl_bandwidth = 100;
     mca_btl_tcp_module.super.btl_latency = 100;
 

ompi/mca/btl/ugni/btl_ugni_component.c

     mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND |
         MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE;
 
+    mca_btl_ugni_module.super.btl_seg_size = sizeof (mca_btl_ugni_segment_t);
+
     mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */
     mca_btl_ugni_module.super.btl_latency   = 2;     /* Microsecs */
 

ompi/mca/btl/ugni/btl_ugni_frag.c

 static inline void mca_btl_ugni_base_frag_constructor (mca_btl_ugni_base_frag_t *frag)
 {
     memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
-    frag->segments[0].seg_addr.pval = frag->base.super.ptr;
+    frag->segments[0].base.seg_addr.pval = frag->base.super.ptr;
 }
 
 static inline void mca_btl_ugni_eager_frag_constructor (mca_btl_ugni_base_frag_t *frag)
         (struct mca_btl_ugni_reg_t *) frag->base.super.registration;
 
     memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
-    frag->segments[0].seg_addr.pval = frag->base.super.ptr;
-    memmove (frag->segments[0].seg_key.key64, &reg->memory_hdl, sizeof (reg->memory_hdl));
+    frag->segments[0].base.seg_addr.pval = frag->base.super.ptr;
+    frag->segments[0].memory_handle = reg->memory_hdl;
 }
 
 OBJ_CLASS_INSTANCE(mca_btl_ugni_smsg_frag_t, mca_btl_base_descriptor_t,

ompi/mca/btl/ugni/btl_ugni_frag.h

 
 typedef void (*frag_cb_t) (struct mca_btl_ugni_base_frag_t *, int);
 
+typedef struct mca_btl_ugni_segment_t {
+    mca_btl_base_segment_t base;
+    gni_mem_handle_t       memory_handle;
+} mca_btl_ugni_segment_t;
+
 typedef struct mca_btl_ugni_base_frag_t {
     mca_btl_base_descriptor_t    base;
     size_t                       hdr_size;
     mca_btl_ugni_frag_hdr_t      hdr;
+    mca_btl_ugni_segment_t       rdma_segments[2];
     mca_btl_base_segment_t       segments[2];
     ompi_common_ugni_post_desc_t post_desc;
     mca_btl_base_endpoint_t     *endpoint;

ompi/mca/btl/ugni/btl_ugni_get.c

 
     des->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
 
-    return mca_btl_ugni_post (frag, true, des->des_dst, des->des_src);
+    return mca_btl_ugni_post (frag, true, (mca_btl_ugni_segment_t *) des->des_dst,
+                              (mca_btl_ugni_segment_t *) des->des_src);
 }
 
 static void mca_btl_ugni_frag_set_ownership (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
     size_t payload_len = frag->hdr.eager.src_seg.seg_len;
     size_t hdr_len = len - payload_len;
     mca_btl_active_message_callback_t *reg;
+    mca_btl_base_segment_t segs[2];
     mca_btl_ugni_base_frag_t tmp;
 
     BTL_VERBOSE(("eager get for rem_ctx %p complete", frag->hdr.eager.ctx));
 
-    tmp.base.des_dst = tmp.segments;
+    tmp.base.des_dst = segs;
     if (hdr_len) {
         tmp.base.des_dst_cnt = 2;
 
-        tmp.segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
-        tmp.segments[0].seg_len       = hdr_len;
-        tmp.segments[1].seg_addr.pval = frag->segments[0].seg_addr.pval;
-        tmp.segments[1].seg_len       = payload_len;
+        segs[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
+        segs[0].seg_len       = hdr_len;
+        segs[1].seg_addr.pval = frag->segments[0].seg_addr.pval;
+        segs[1].seg_len       = payload_len;
     } else {
         tmp.base.des_dst_cnt = 1;
 
-        tmp.segments[0].seg_addr.pval = frag->segments[0].seg_addr.pval;
-        tmp.segments[0].seg_len       = payload_len;
+        segs[0].seg_addr.pval = frag->segments[0].seg_addr.pval;
+        segs[0].seg_len       = payload_len;
     }
 
     reg = mca_btl_base_active_message_trigger + tag;
         frag->base.des_src_cnt = 1;
 
         /* increase size to a multiple of 4 bytes (required for get) */
-        frag->segments[0].seg_len = frag->segments[1].seg_len =
+        frag->segments[0].base.seg_len = frag->segments[1].base.seg_len =
             (hdr.eager.src_seg.seg_len + 3) & ~3;
 
         rc = mca_btl_ugni_post (frag, true, frag->base.des_dst, frag->base.des_src);

ompi/mca/btl/ugni/btl_ugni_module.c

     frag->base.des_dst = frag->segments + 1;
     frag->base.des_dst_cnt = 1;
 
-    frag->segments[0].seg_addr.pval = NULL;
-    frag->segments[0].seg_len       = 0;
-    frag->segments[1].seg_addr.pval = frag->base.super.ptr;
-    frag->segments[1].seg_len       = size;
+    frag->segments[0].base.seg_addr.pval = NULL;
+    frag->segments[0].base.seg_len       = 0;
+    frag->segments[1].base.seg_addr.pval = frag->base.super.ptr;
+    frag->segments[1].base.seg_len       = size;
 
     frag->flags = MCA_BTL_UGNI_FRAG_BUFFERED;
     if (size > mca_btl_ugni_component.smsg_max_data) {
 
         registration = (mca_btl_ugni_reg_t *) frag->base.super.registration;
 
-        memcpy ((void *) frag->segments[1].seg_key.key64,
-                (void *)&registration->memory_hdl,
-                sizeof (registration->memory_hdl));
+        frag->segments[1].memory_handle = registration->memory_hdl;
     } else {
         frag->hdr_size = sizeof (frag->hdr.send);
     }
         frag->registration = (mca_btl_ugni_reg_t*) registration;
     }
 
-    memcpy ((void *) frag->segments[0].seg_key.key64,
-            (void *)&((mca_btl_ugni_reg_t *)registration)->memory_hdl,
-            sizeof (((mca_btl_ugni_reg_t *)registration)->memory_hdl));
-
-    frag->segments[0].seg_len = *size;
-    frag->segments[0].seg_addr.pval = data_ptr;
+    frag->segments[0].memory_handle      = ((mca_btl_ugni_reg_t *)registration)->memory_hdl;
+    frag->segments[0].base.seg_len       = *size;
+    frag->segments[0].base.seg_addr.pval = data_ptr;
 
     frag->base.des_dst     = frag->segments;
     frag->base.des_dst_cnt = 1;

ompi/mca/btl/ugni/btl_ugni_prepare.h

         frag->flags = MCA_BTL_UGNI_FRAG_EAGER | MCA_BTL_UGNI_FRAG_IGNORE;
 
         frag->registration = registration;
-        memcpy ((void *) frag->segments[1].seg_key.key64,
-                (void *)&registration->memory_hdl,
-                sizeof (registration->memory_hdl));        
+        frag->segments[1].memory_handle = registration->memory_hdl;
 
         frag->hdr_size = reserve + sizeof (frag->hdr.eager);
-        frag->segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
+        frag->segments[0].base.seg_addr.pval = frag->hdr.eager_ex.pml_header;
     } else {
         frag->hdr_size = reserve + sizeof (frag->hdr.send);
-        frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
+        frag->segments[0].base.seg_addr.pval = frag->hdr.send_ex.pml_header;
     }
 
-    frag->segments[0].seg_len       = reserve;
+    frag->segments[0].base.seg_len       = reserve;
 
-    frag->segments[1].seg_addr.pval = data_ptr;
-    frag->segments[1].seg_len       = *size;
+    frag->segments[1].base.seg_addr.pval = data_ptr;
+    frag->segments[1].base.seg_len       = *size;
 
     frag->base.des_src     = frag->segments;
     frag->base.des_src_cnt = 2;
 
         registration = (mca_btl_ugni_reg_t *) frag->base.super.registration;
 
-        memcpy ((void *) frag->segments[1].seg_key.key64,
-                (void *)&registration->memory_hdl,
-                sizeof (registration->memory_hdl));
+        frag->segments[1].memory_handle = registration->memory_hdl;
 
         frag->hdr_size = reserve + sizeof (frag->hdr.eager);
-        frag->segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
+        frag->segments[0].base.seg_addr.pval = frag->hdr.eager_ex.pml_header;
     } else {
         (void) MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
         if (OPAL_UNLIKELY(NULL == frag)) {
         }
 
         frag->hdr_size = reserve + sizeof (frag->hdr.send);
-        frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
+        frag->segments[0].base.seg_addr.pval = frag->hdr.send_ex.pml_header;
     }
 
     frag->flags |= MCA_BTL_UGNI_FRAG_BUFFERED;
         return NULL;
     }
 
-    frag->segments[0].seg_len       = reserve;
+    frag->segments[0].base.seg_len       = reserve;
 
-    frag->segments[1].seg_addr.pval = frag->base.super.ptr;
-    frag->segments[1].seg_len       = *size;
+    frag->segments[1].base.seg_addr.pval = frag->base.super.ptr;
+    frag->segments[1].base.seg_len       = *size;
 
     frag->base.des_src     = frag->segments;
     frag->base.des_src_cnt = 2;
             frag->registration = (mca_btl_ugni_reg_t *) registration;
         }
 
-        memcpy ((void *) frag->segments[0].seg_key.key64,
-                (void *)&((mca_btl_ugni_reg_t *)registration)->memory_hdl,
-                sizeof (((mca_btl_ugni_reg_t *)registration)->memory_hdl));
+        frag->segments[0].memory_handle = ((mca_btl_ugni_reg_t *)registration)->memory_hdl;
     } else {
-        memset ((void *) frag->segments[0].seg_key.key64, 0,
-                sizeof (frag->segments[0].seg_key.key64));
+        memset ((void *) &frag->segments[0].memory_handle, 0,
+                sizeof (frag->segments[0].memory_handle));
     }
 
-    frag->segments[0].seg_addr.pval = data_ptr;
-    frag->segments[0].seg_len       = *size;
+    frag->segments[0].base.seg_addr.pval = data_ptr;
+    frag->segments[0].base.seg_len       = *size;
 
     frag->base.des_src     = frag->segments;
     frag->base.des_src_cnt = 1;

ompi/mca/btl/ugni/btl_ugni_put.c

 
     des->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
 
-    return mca_btl_ugni_post (frag, false, des->des_src, des->des_dst);
+    return mca_btl_ugni_post (frag, false, (mca_btl_ugni_segment_t *) des->des_src,
+                              (mca_btl_ugni_segment_t *) des->des_dst);
 }

ompi/mca/btl/ugni/btl_ugni_rdma.h

 static inline void init_gni_post_desc (mca_btl_ugni_base_frag_t *frag,
                                       gni_post_type_t op_type,
                                       uint64_t lcl_addr, 
-                                      gni_mem_handle_t *lcl_mdh, 
+                                      gni_mem_handle_t lcl_mdh, 
                                       uint64_t rem_addr,
-                                      gni_mem_handle_t *rem_mdh,
+                                      gni_mem_handle_t rem_mdh,
                                       uint64_t bufsize,
                                       gni_cq_handle_t cq_hndl) {
     frag->post_desc.base.type            = op_type;
     frag->post_desc.base.cq_mode         = GNI_CQMODE_GLOBAL_EVENT;
     frag->post_desc.base.dlvr_mode       = GNI_DLVMODE_PERFORMANCE;
     frag->post_desc.base.local_addr      = (uint64_t) lcl_addr;
-    frag->post_desc.base.local_mem_hndl  = *lcl_mdh;
+    frag->post_desc.base.local_mem_hndl  = lcl_mdh;
     frag->post_desc.base.remote_addr     = (uint64_t) rem_addr;
-    frag->post_desc.base.remote_mem_hndl = *rem_mdh;
+    frag->post_desc.base.remote_mem_hndl = rem_mdh;
     frag->post_desc.base.length          = bufsize;
     frag->post_desc.base.rdma_mode       = 0;
     frag->post_desc.base.src_cq_hndl     = cq_hndl;
 }
 
 static inline int mca_btl_ugni_post_fma (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
-                                         mca_btl_base_segment_t *lcl_seg, mca_btl_base_segment_t *rem_seg)
+                                         mca_btl_ugni_segment_t *lcl_seg, mca_btl_ugni_segment_t *rem_seg)
 {
     gni_return_t rc;
 
-    /* Post descriptor */
-    init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
-                        (gni_mem_handle_t *)&lcl_seg->seg_key.key64,
-                        rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
-                        lcl_seg->seg_len, 0); /* CQ is ignored for FMA transactions */
+    /* Post descriptor (CQ is ignored for FMA transactions) */
+    init_gni_post_desc (frag, op_type, lcl_seg->base.seg_addr.lval, lcl_seg->memory_handle,
+                        rem_seg->base.seg_addr.lval, rem_seg->memory_handle, lcl_seg->base.seg_len, 0);
 
     rc = GNI_PostFma (frag->endpoint->rdma_ep_handle, &frag->post_desc.base);
     if (GNI_RC_SUCCESS != rc) {
 }
 
 static inline int mca_btl_ugni_post_bte (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
-                                         mca_btl_base_segment_t *lcl_seg, mca_btl_base_segment_t *rem_seg)
+                                         mca_btl_ugni_segment_t *lcl_seg, mca_btl_ugni_segment_t *rem_seg)
 {
     gni_return_t rc;
 
     /* Post descriptor */
-    init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
-                        (gni_mem_handle_t *)&lcl_seg->seg_key.key64,
-                        rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
-                        lcl_seg->seg_len, frag->endpoint->btl->rdma_local_cq);
+    init_gni_post_desc (frag, op_type, lcl_seg->base.seg_addr.lval, lcl_seg->memory_handle,
+                        rem_seg->base.seg_addr.lval, rem_seg->memory_handle, lcl_seg->base.seg_len,
+                        frag->endpoint->btl->rdma_local_cq);
 
     rc = GNI_PostRdma (frag->endpoint->rdma_ep_handle, &frag->post_desc.base);
     if (GNI_RC_SUCCESS != rc) {
     return OMPI_SUCCESS;
 }
 
-static inline int mca_btl_ugni_post (mca_btl_ugni_base_frag_t *frag, bool get, mca_btl_base_segment_t *lcl_seg,
-                                     mca_btl_base_segment_t *rem_seg) {
+static inline int mca_btl_ugni_post (mca_btl_ugni_base_frag_t *frag, bool get, mca_btl_ugni_segment_t *lcl_seg,
+                                     mca_btl_ugni_segment_t *rem_seg) {
     frag->cbfunc = mca_btl_ugni_frag_complete;
 
     if (frag->base.des_src->seg_len <= mca_btl_ugni_component.ugni_fma_limit) {

ompi/mca/btl/ugni/btl_ugni_smsg.c

 {
     mca_btl_active_message_callback_t *reg;
     mca_btl_ugni_base_frag_t frag;
+    mca_btl_base_segment_t seg;
     bool disconnect = false;
     uintptr_t data_ptr;
     gni_return_t rc;
             BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}", len, tag));
 
             reg = mca_btl_base_active_message_trigger + tag;
-            frag.base.des_dst     = frag.segments;
+            frag.base.des_dst     = &seg;
             frag.base.des_dst_cnt = 1;
 
-            frag.segments[0].seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t));
-            frag.segments[0].seg_len       = len;
+            seg.seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t));
+            seg.seg_len       = len;
 
             assert (NULL != reg->cbfunc);
 

ompi/mca/btl/ugni/btl_ugni_smsg.h

                                           mca_btl_ugni_base_frag_t *frag) {
     if (OPAL_LIKELY(!(frag->flags & MCA_BTL_UGNI_FRAG_EAGER))) {
         return ompi_mca_btl_ugni_smsg_send (frag, &frag->hdr.send, frag->hdr_size,
-                                            frag->segments[1].seg_addr.pval, frag->segments[1].seg_len,
+                                            frag->segments[1].base.seg_addr.pval,
+                                            frag->segments[1].base.seg_len,
                                             MCA_BTL_UGNI_TAG_SEND);
     }
 

ompi/mca/btl/vader/btl_vader.c

     
     opal_convertor_get_current_pointer (convertor, (void **) &data_ptr);
 
-    frag->segment.seg_key.key64[0] = (uint64_t)(uintptr_t) data_ptr;
-    frag->segment.seg_len     = *size;
+    frag->segment.seg_addr.pval = data_ptr;
+    frag->segment.seg_len       = *size;
     
     frag->base.des_dst     = &frag->segment;
     frag->base.des_dst_cnt = 1;
             return NULL;
         }
 
-        frag->segment.seg_key.key64[0] = (uint64_t)(uintptr_t) data_ptr;
-        frag->segment.seg_len = reserve + *size;
+        frag->segment.seg_addr.pval = data_ptr;
+        frag->segment.seg_len       = reserve + *size;
     }
 
     frag->base.des_src     = &frag->segment;

ompi/mca/btl/vader/btl_vader_component.c

     mca_btl_vader.super.btl_min_rdma_pipeline_size = mca_btl_vader.super.btl_eager_limit;
     mca_btl_vader.super.btl_flags = MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_PUT |
         MCA_BTL_FLAGS_SEND_INPLACE;
+    mca_btl_vader.super.btl_seg_size = sizeof (mca_btl_base_segment_t);
 
     mca_btl_vader.super.btl_bandwidth = 40000; /* Mbs */
     mca_btl_vader.super.btl_latency   = 1;     /* Microsecs */

ompi/mca/btl/vader/btl_vader_get.c

     mca_mpool_base_registration_t *reg;
     void *rem_ptr;
 
-    reg = vader_get_registation (endpoint->peer_smp_rank,
-                                 (void *)(uintptr_t) src->seg_key.key64[0],
+    reg = vader_get_registation (endpoint->peer_smp_rank, src->seg_addr.pval,
                                  src->seg_len, 0);
     if (OPAL_UNLIKELY(NULL == reg)) {
         return OMPI_ERROR;
     }
 
-    rem_ptr = vader_reg_to_ptr (reg, (void *)(uintptr_t) src->seg_key.key64[0]);
+    rem_ptr = vader_reg_to_ptr (reg, src->seg_addr.pval);
 
-    vader_memmove ((void *)(uintptr_t) dst->seg_key.key64[0], rem_ptr, size);
+    vader_memmove (dst->seg_addr.pval, rem_ptr, size);
 
     vader_return_registration (reg, endpoint->peer_smp_rank);
 

ompi/mca/btl/vader/btl_vader_put.c

     mca_mpool_base_registration_t *reg;
     void *rem_ptr;
 
-    reg = vader_get_registation (endpoint->peer_smp_rank,
-                                 (void *)(uintptr_t) dst->seg_key.key64[0],
+    reg = vader_get_registation (endpoint->peer_smp_rank, dst->seg_addr.pval,
                                  dst->seg_len, 0);
     if (OPAL_UNLIKELY(NULL == reg)) {
         return OMPI_ERROR;
     }
 
-    rem_ptr = vader_reg_to_ptr (reg, (void *)(uintptr_t) dst->seg_key.key64[0]);
+    rem_ptr = vader_reg_to_ptr (reg, dst->seg_addr.pval);
 
-    vader_memmove (rem_ptr, (void *)(uintptr_t) src->seg_key.key64[0], size);
+    vader_memmove (rem_ptr, src->seg_addr.pval, size);
 
     vader_return_registration (reg, endpoint->peer_smp_rank);
 

ompi/mca/osc/rdma/osc_rdma.h

 
 
 struct ompi_osc_rdma_btl_t {
-    uint64_t peer_seg_key;
+    uint8_t  peer_seg[MCA_BTL_SEG_MAX_SIZE];
     mca_bml_base_btl_t *bml_btl;
     int      rdma_order;
     int32_t  num_sent;

ompi/mca/osc/rdma/osc_rdma_component.c

+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University.
  *                         All rights reserved.
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2007-2012 Los Alamos National Security, LLC.  All rights
  *                         reserved. 
  * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
                 index = module->m_peer_info[origin].peer_num_btls++;
                 rdma_btl = &(module->m_peer_info[origin].peer_btls[index]);
 
-                rdma_btl->peer_seg_key = header->hdr_segkey;
+                memmove (rdma_btl->peer_seg, header + 1, bml_btl->btl->btl_seg_size);
+
                 rdma_btl->bml_btl = bml_btl;
                 rdma_btl->rdma_order = MCA_BTL_NO_ORDER;
                 rdma_btl->num_sent = 0;
     ompi_osc_rdma_module_t *module;
     ompi_proc_t *proc;
     mca_bml_base_btl_t *bml_btl;
-    uint64_t seg_key;
+    void *seg;
 };
 typedef struct peer_rdma_send_info_t peer_rdma_send_info_t;
 OBJ_CLASS_INSTANCE(peer_rdma_send_info_t, opal_list_item_t, NULL, NULL);
     mca_bml_base_btl_t *bml_btl = NULL;
     mca_btl_base_descriptor_t *descriptor = NULL;
     ompi_osc_rdma_rdma_info_header_t *header = NULL;
+    size_t hdr_size;
         
     bml_btl = peer_send_info->bml_btl;
-    mca_bml_base_alloc(bml_btl, &descriptor, MCA_BTL_NO_ORDER,
-                       sizeof(ompi_osc_rdma_rdma_info_header_t),
+
+    hdr_size = sizeof(ompi_osc_rdma_rdma_info_header_t) + bml_btl->btl->btl_seg_size;
+
+    mca_bml_base_alloc(bml_btl, &descriptor, MCA_BTL_NO_ORDER, hdr_size,
                        MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
     if (NULL == descriptor) {
         ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE;
     }
 
     /* verify at least enough space for header */
-    if (descriptor->des_src[0].seg_len < sizeof(ompi_osc_rdma_rdma_info_header_t)) {
+    if (descriptor->des_src[0].seg_len < hdr_size) {
         ret = OMPI_ERR_OUT_OF_RESOURCE;
         goto cleanup;
     }
     header = (ompi_osc_rdma_rdma_info_header_t*) descriptor->des_src[0].seg_addr.pval;
     header->hdr_base.hdr_type = OMPI_OSC_RDMA_HDR_RDMA_INFO;
     header->hdr_base.hdr_flags = 0;
-    header->hdr_segkey = peer_send_info->seg_key;
     header->hdr_origin = ompi_comm_rank(module->m_comm);
     header->hdr_windx = ompi_comm_get_cid(module->m_comm);
 
+    memmove (header + 1, peer_send_info->seg, bml_btl->btl->btl_seg_size);
+
 #ifdef WORDS_BIGENDIAN
     header->hdr_base.hdr_flags |= OMPI_OSC_RDMA_HDR_FLAG_NBO;
 #elif OPAL_ENABLE_HETEROGENEOUS_SUPPORT
             peer_send_info->module = module;
             peer_send_info->proc = ompi_comm_peer_lookup(module->m_comm, i);
             peer_send_info->bml_btl = peer_info->local_btls[j];
-            peer_send_info->seg_key = 
-                peer_info->local_descriptors[j]->des_dst[0].seg_key.key64[0];
+            peer_send_info->seg = (void *) peer_info->local_descriptors[j]->des_dst;
 
             ret = rdma_send_info_send(module, peer_send_info);
             if (OMPI_SUCCESS != ret) {

ompi/mca/osc/rdma/osc_rdma_data_move.c

 
             descriptor->des_dst = sendreq->remote_segs;
             descriptor->des_dst_cnt = 1;
+	    memmove (descriptor->des_dst, rdma_btl->peer_seg, sizeof (rdma_btl->peer_seg));
+
             descriptor->des_dst[0].seg_addr.lval = 
                 module->m_peer_info[target].peer_base + 
                 ((unsigned long)sendreq->req_target_disp * module->m_win->w_disp_unit);
             descriptor->des_dst[0].seg_len = 
                 sendreq->req_origin_bytes_packed;
-            descriptor->des_dst[0].seg_key.key64[0] = 
-                rdma_btl->peer_seg_key;
 #if 0
             opal_output(0, "putting to %d: 0x%lx(%d), %d, %d",
                         target, descriptor->des_dst[0].seg_addr.lval,
 
             descriptor->des_src = sendreq->remote_segs;
             descriptor->des_src_cnt = 1;
+	    memmove (descriptor->des_src, rdma_btl->peer_seg, sizeof (rdma_btl->peer_seg));
+
             descriptor->des_src[0].seg_addr.lval = 
                 module->m_peer_info[target].peer_base + 
                 ((unsigned long)sendreq->req_target_disp * module->m_win->w_disp_unit);