Commits

Nathan Hjelm committed c8949be

finish removing btl segment keys from mca_btl_base_segment_t

Comments (0)

Files changed (44)

ompi/mca/btl/btl.h

  * completion function, this implies that all data payload in the 
  * mca_btl_base_descriptor_t must be copied out within this callback or 
  * forfeited back to the BTL.
+ * Note also that descriptor segments (des_dst, des_src) must be base
+ * segments for all callbacks.
  * 
  * @param[IN] btl        BTL module
  * @param[IN] tag        The active message receive callback tag value 

ompi/mca/btl/elan/btl_elan.c

         0, /* latency */
         0, /* bandwidth */
         0, /* flags */
+        0, /* segment size */
         mca_btl_elan_add_procs,
         mca_btl_elan_del_procs,
         NULL,  /* btl_register */

ompi/mca/btl/elan/btl_elan_component.c

     mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
     mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128 * 1024;
     mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND;
+    mca_btl_elan_module.super.btl_seg_size = sizeof (mca_btl_base_segment_t);
     mca_btl_elan_module.super.btl_bandwidth = 1959;
     mca_btl_elan_module.super.btl_latency = 4;
     mca_btl_base_param_register(&mca_btl_elan_component.super.btl_version,

ompi/mca/btl/mx/btl_mx.c

         0, /* exclusivity */
         0, /* latency */
         0, /* bandwidth */
+        0, /* segment size */
         MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_PUT, /* flags */
         mca_btl_mx_add_procs,
         mca_btl_mx_del_procs,

ompi/mca/btl/ofud/btl_ofud.c

         0, /* latency */
         0, /* bandwidth */
         MCA_BTL_FLAGS_SEND,
+        0, /* segment length */
         mca_btl_ud_add_procs,
         mca_btl_ud_del_procs,
         NULL,

ompi/mca/btl/ofud/btl_ofud_component.c

                              "Approximate maximum bandwidth of interconnect",
                              800, (int*)&mca_btl_ofud_module.super.btl_bandwidth);
 
+    mca_btl_ofud_module.super.btl_seg_size = sizeof (mca_btl_base_segment_t);
+
     mca_btl_ofud_module.super.btl_eager_limit -= sizeof(mca_btl_ud_header_t);
     mca_btl_ofud_module.super.btl_max_send_size -= sizeof(mca_btl_ud_header_t);
 

ompi/mca/btl/openib/btl_openib.c

         0, /* latency */
         0, /* bandwidth */
         0, /* TODO this should be PUT btl flags */
+        0, /* segment size */
         mca_btl_openib_add_procs,
         mca_btl_openib_del_procs,
         NULL,

ompi/mca/btl/portals/btl_portals.c

         0,   /* latency */
         0,   /* bandwidth */
         0,   /* btl flags */
+        0,   /* btl segment size */
 
         mca_btl_portals_add_procs,
         mca_btl_portals_del_procs,
         ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
                           OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                           *((mca_btl_base_endpoint_t*) peer),
-			  frag->segments[0].key, /* match */
+                          frag->segments[0].key, /* match */
                           0, /* ignore */
                           PTL_UNLINK,
                           PTL_INS_AFTER,
                          "rdma dest posted for frag 0x%lx, callback 0x%lx, bits %" PRIu64 " flags %d",
                          (unsigned long) frag,
                          (unsigned long) frag->base.des_cbfunc,
-			 frag->segments[0].key;
+                         frag->segments[0].key;
                          flags));
 
     /* create a match entry */
     ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
                       OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                       *((mca_btl_base_endpoint_t*) peer),
-		      frag->segments[0].key, /* match */
+                      frag->segments[0].key, /* match */
                       0, /* ignore */
                       PTL_UNLINK,
                       PTL_INS_AFTER,

ompi/mca/btl/portals/btl_portals_component.c

                                              ));
                         
                         OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,"received %d bytes \n", (int) ev.mlength));
-			frag->base.des_dst = seg;
-			seg[0].seg_addr.pval = &frag->data;
-			seg[0].seg_len = header_size;
+                        frag->base.des_dst = seg;
+                        seg[0].seg_addr.pval = &frag->data;
+                        seg[0].seg_len = header_size;
                         if(ev.mlength) {
-			    seg[1].seg_addr.pval = ((((char*) ev.md.start) + ev.offset));
-			    seg[1].seg_len = ev.mlength;
+                            seg[1].seg_addr.pval = ((((char*) ev.md.start) + ev.offset));
+                            seg[1].seg_len = ev.mlength;
                             frag->base.des_dst_cnt = 2;
                         } else { 
                             frag->base.des_dst_cnt = 1;
                     } else { 
                         /* if we ever make this thread hot, need to do
                            something with the receive fragments */
-                        frag->segments[0].seg_addr.pval = (((char*) ev.md.start) + ev.offset);
-                        frag->segments[0].seg_len = ev.mlength;
+                        seg[0].seg_addr.pval = (((char*) ev.md.start) + ev.offset);
+                        seg[0].seg_len = ev.mlength;
                         
                         OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
                                              "received send fragment 0x%lx (thresh: %d, length %d)", 
                         block->full = true;
                     }
 
+                    /* NTH: is it ok to overwrite this. All callbacks should expect base segments */
+                    frag->base.des_dst = seg;
+
                     mca_btl_base_active_message_trigger[tag].cbfunc(
                                              &mca_btl_portals_module.super,
                                              tag,

ompi/mca/btl/portals/btl_portals_frag.c

     frag->base.des_src = frag->segments;
     frag->base.des_src_cnt = 2;
 
-    frag->segments[0].seg_addr.pval = frag + 1;
-    frag->segments[0].seg_len = frag->size;
-    frag->segments[0].seg_key.key64[0] = 0;
+    frag->segments[0].base.seg_addr.pval = frag + 1;
+    frag->segments[0].base.seg_len = frag->size;
+    frag->segments[0].key = 0;
 
     frag->md_h = PTL_INVALID_HANDLE;
 }

ompi/mca/btl/portals/btl_portals_rdma.c

                     struct mca_btl_base_endpoint_t* btl_peer,
                     struct mca_btl_base_descriptor_t* descriptor)
 {
+    mca_btl_portals_segment_t *dst_seg = (mca_btl_portals_segment_t *) descriptor->des_dst;
     mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
     int ret;
     unsigned char hdr_data[8];
 
     OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
                          "PtlPut (rdma) fragment %lx, bits %" PRIx64,
-                         (unsigned long) frag,
-                         frag->base.des_dst[0].seg_key.key64[0]));
+                         (unsigned long) frag, dst_seg->key));
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
     assert(frag->md_h != PTL_INVALID_HANDLE);
                  *((mca_btl_base_endpoint_t*) btl_peer),
                  OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                  0, /* ac_index - not used*/
-                 frag->base.des_dst[0].seg_key.key64[0], /* match bits */
+                 dst_seg->key, /* match bits */
                  0, /* remote offset - not used */
                  *((ptl_hdr_data_t*) hdr_data));            /* hdr_data: tag */
     if (ret != PTL_OK) {
                     struct mca_btl_base_endpoint_t* btl_peer,
                     struct mca_btl_base_descriptor_t* descriptor)
 {
+    mca_btl_portals_segment_t *src_seg = (mca_btl_portals_segment_t *) descriptor->des_src;
     mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
     int ret;
 
     OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
                          "PtlGet (rdma) fragment %lx, bits %" PRIx64,
-                         (unsigned long) frag,
-                         frag->base.des_src[0].seg_key.key64[0]));
+                         (unsigned long) frag, src_seg->key));
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
     assert(frag->md_h != PTL_INVALID_HANDLE);
                  *((mca_btl_base_endpoint_t*) btl_peer),
                  OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                  0, /* ac_index - not used*/
-                 frag->base.des_src[0].seg_key.key64[0], /* match bits */
+                 src_seg->key, /* match bits */
                  0); /* remote offset - not used */
     if (ret != PTL_OK) {
         opal_output(mca_btl_portals_component.portals_output,

ompi/mca/btl/portals/btl_portals_send.c

 
     if (frag->md_h == PTL_INVALID_HANDLE) {
         /* setup the send - always describe entire fragment */
-        mca_btl_portals_module.md_send.start = frag->segments[0].seg_addr.pval;
+        mca_btl_portals_module.md_send.start = frag->segments[0].base.seg_addr.pval;
         mca_btl_portals_module.md_send.length = 
-            0 == frag->size ? frag->segments[0].seg_len : frag->size;
+            0 == frag->size ? frag->segments[0].base.seg_len : frag->size;
 #if OPAL_ENABLE_DEBUG 
         mca_btl_portals_module.md_send.options = 
             PTL_MD_EVENT_START_DISABLE;
                          "fragment info:\n"
                          "\tstart: 0x%lx\n"
                          "\tlen: %d",
-                         (unsigned long) frag->segments[0].seg_addr.pval,
-                         frag->segments[0].seg_len)); 
+                         (unsigned long) frag->segments[0].base.seg_addr.pval,
+                         frag->segments[0].base.seg_len)); 
     
-    ret = PtlPutRegion(frag->md_h,                /* memory descriptor */
-                       0,                         /* fragment offset */
-                       frag->segments[0].seg_len, /* fragment length */
+    ret = PtlPutRegion(frag->md_h,                     /* memory descriptor */
+                       0,                              /* fragment offset */
+                       frag->segments[0].base.seg_len, /* fragment length */
                        (mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
                        *((mca_btl_base_endpoint_t*) endpoint),
                        OMPI_BTL_PORTALS_SEND_TABLE_ID,
-                       0,                         /* ac_index - not used */
-                       0,                         /* match bits */
-                       0,                         /* remote offset - not used */
-                       *((ptl_hdr_data_t*) hdr_data));            /* hdr_data: tag */
+                       0,                              /* ac_index - not used */
+                       0,                              /* match bits */
+                       0,                              /* remote offset - not used */
+                       *((ptl_hdr_data_t*) hdr_data)); /* hdr_data: tag */
     if (ret != PTL_OK) {
         opal_output(mca_btl_portals_component.portals_output,
                     "send: PtlPut failed with error %d", ret);
         OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, -1);
         return OMPI_ERR_RESOURCE_BUSY;
     }
-    frag->segments[0].seg_len = payload_size;
+    frag->segments[0].base.seg_len = payload_size;
     frag->base.des_src_cnt = 1;
     frag->base.des_flags = flags; 
     frag->base.order = MCA_BTL_NO_ORDER;
 
     if(payload_size) { 
         /* pack the data into the supplied buffer */
-        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segments[0].seg_addr.pval);
+        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segments[0].base.seg_addr.pval);
         iov.iov_len  = max_data = payload_size;
         iov_count    = 1;
         
 
     if (frag->md_h == PTL_INVALID_HANDLE) {
         /* setup the send - always describe entire fragment */
-        mca_btl_portals_module.md_send.start = frag->segments[0].seg_addr.pval;
+        mca_btl_portals_module.md_send.start = frag->segments[0].base.seg_addr.pval;
         mca_btl_portals_module.md_send.length = 
-            0 == frag->size ? frag->segments[0].seg_len : frag->size;
+            0 == frag->size ? frag->segments[0].base.seg_len : frag->size;
 #if OPAL_ENABLE_DEBUG 
         mca_btl_portals_module.md_send.options = 
             PTL_MD_EVENT_START_DISABLE;
                          "fragment info:\n"
                          "\tstart: 0x%lx\n"
                          "\tlen: %d",
-                         (unsigned long) frag->segments[0].seg_addr.pval,
-                         frag->segments[0].seg_len)); 
+                         (unsigned long) frag->segments[0].base.seg_addr.lval,
+                         frag->segments[0].seg_len));
     
-    ret = PtlPutRegion(frag->md_h,                /* memory descriptor */
-                       0,                         /* fragment offset */
-                       frag->segments[0].seg_len, /* fragment length */
+    ret = PtlPutRegion(frag->md_h,                     /* memory descriptor */
+                       0,                              /* fragment offset */
+                       frag->segments[0].base.seg_len, /* fragment length */
                        (mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ),
                        *((mca_btl_base_endpoint_t*) endpoint),
                        OMPI_BTL_PORTALS_SEND_TABLE_ID,
-                       0,                         /* ac_index - not used */
+                       0,                              /* ac_index - not used */
                        *((ptl_match_bits_t*) match_bits),                /* match bits */
-                       0,                         /* remote offset - not used */
-                       *((ptl_hdr_data_t*) hdr_data));            /* hdr_data: tag */
+                       0,                              /* remote offset - not used */
+                       *((ptl_hdr_data_t*) hdr_data)); /* hdr_data: tag */
     
     if (ret != PTL_OK) {
         opal_output(mca_btl_portals_component.portals_output,

ompi/mca/btl/sctp/btl_sctp.c

         0, /* latency */
         0, /* bandwidth */
         0, /* flags */
+        0, /* segment size */
         mca_btl_sctp_add_procs,
         mca_btl_sctp_del_procs,
         NULL, 

ompi/mca/btl/sctp/btl_sctp_component.c

                                        MCA_BTL_FLAGS_NEED_CSUM | 
                                        MCA_BTL_FLAGS_NEED_ACK |
                                        MCA_BTL_FLAGS_HETEROGENEOUS_RDMA;
+    mca_btl_sctp_module.super.btl_seg_size = sizeof (mca_btl_base_segment_t);
     mca_btl_sctp_module.super.btl_bandwidth = 100;
     mca_btl_sctp_module.super.btl_latency = 100;
     mca_btl_base_param_register(&mca_btl_sctp_component.super.btl_version,

ompi/mca/btl/self/btl_self.c

     0, /* btl_latency */
     0, /* btl_bandwidth */
     0, /* btl flags */
+    0, /* btl segment size */
     mca_btl_self_add_procs,
     mca_btl_self_del_procs,
     NULL,

ompi/mca/btl/self/btl_self_component.c

     mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;
     mca_btl_self.btl_min_rdma_pipeline_size = 0;
     mca_btl_self.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE;
-    mca_btl_self.btl_seg_len = sizeof (mca_btl_base_segment_t);
+    mca_btl_self.btl_seg_size = sizeof (mca_btl_base_segment_t);
     mca_btl_self.btl_bandwidth = 100;
     mca_btl_self.btl_latency = 0;
     mca_btl_base_param_register(&mca_btl_self_component.super.btl_version,

ompi/mca/btl/sm/btl_sm.c

         0, /* btl_latency */
         0, /* btl_bandwidth */
         0, /* btl flags */
+        0, /* btl segment size */
         mca_btl_sm_add_procs,
         mca_btl_sm_del_procs,
         NULL,
         }
         iov.iov_len = max_data;
         iov.iov_base =
-	    (IOVBASE_TYPE*)(frag->segment.base.seg_addr.lval + reserve);
+            (IOVBASE_TYPE*)(frag->segment.base.seg_addr.lval + reserve);
 
         rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
         if( OPAL_UNLIKELY(rc < 0) ) {
 #if 0
 #define MCA_BTL_SM_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)          \
     do {                                                                \
-	char* _memory = (char*)(sm_frag)->segment.base.seg_addr.pval +	\
-            (sm_frag)->segment.base.seg_len;				\
+        char* _memory = (char*)(sm_frag)->segment.base.seg_addr.pval +  \
+            (sm_frag)->segment.base.seg_len;                            \
         int* _intmem;                                                   \
         size_t align = (intptr_t)_memory & 0xFUL;                       \
         switch( align & 0x3 ) {                                         \

ompi/mca/btl/sm/btl_sm_component.c

     }
 
 #endif
-    mca_btl_sm.super.seg_len = sizeof (mca_btl_sm_segment_t);
+    mca_btl_sm.super.btl_seg_size = sizeof (mca_btl_sm_segment_t);
     mca_btl_sm.super.btl_bandwidth = 9000;  /* Mbs */
     mca_btl_sm.super.btl_latency   = 1;     /* Microsecs */
 
 int mca_btl_sm_component_progress(void)
 {
     /* local variables */
+    mca_btl_base_segment_t seg;
     mca_btl_sm_frag_t *frag;
     mca_btl_sm_frag_t Frag;
     sm_fifo_t *fifo = NULL;
 #endif
                 /* recv upcall */
                 reg = mca_btl_base_active_message_trigger + hdr->tag;
-                Frag.segment.seg_addr.pval = ((char*)hdr) +
-                    sizeof(mca_btl_sm_hdr_t);
-                Frag.segment.seg_len = hdr->len;
+                seg.seg_addr.pval = ((char *)hdr) + sizeof(mca_btl_sm_hdr_t);
+                seg.seg_len = hdr->len;
                 Frag.base.des_dst_cnt = 1;
-                Frag.base.des_dst = &(Frag.segment);
+                Frag.base.des_dst = &seg;
                 reg->cbfunc(&mca_btl_sm.super, hdr->tag, &(Frag.base),
                             reg->cbdata);
                 /* return the fragment */

ompi/mca/btl/sm/btl_sm_frag.c

         frag->hdr->my_smp_rank = mca_btl_sm_component.my_smp_rank;
     }
     frag->segment.base.seg_len = frag->size;
-    frag->base.des_src = &frag->segment;
+    frag->base.des_src = &frag->segment.base;
     frag->base.des_src_cnt = 1;
-    frag->base.des_dst = &frag->segment;
+    frag->base.des_dst = &frag->segment.base;
     frag->base.des_dst_cnt = 1;
     frag->base.des_flags = 0;
 }

ompi/mca/btl/smcuda/btl_smcuda.c

         0, /* btl_latency */
         0, /* btl_bandwidth */
         0, /* btl flags */
+        0, /* btl segment size */
         mca_btl_smcuda_add_procs,
         mca_btl_smcuda_del_procs,
         NULL,
         }
         iov.iov_len = max_data;
         iov.iov_base =
-	    (IOVBASE_TYPE*)(frag->segment.base.seg_addr.lval + reserve);
+            (IOVBASE_TYPE*)(frag->segment.base.seg_addr.lval + reserve);
 
         rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
         if( OPAL_UNLIKELY(rc < 0) ) {
 }
 
 #if 0
-#define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)	\
+#define MCA_BTL_SMCUDA_TOUCH_DATA_TILL_CACHELINE_BOUNDARY(sm_frag)      \
     do {                                                                \
-        char* _memory = (char*)(sm_frag)->segment.base.seg_addr.pval +	\
-            (sm_frag)->segment.base.seg_len;				\
+        char* _memory = (char*)(sm_frag)->segment.base.seg_addr.pval +  \
+            (sm_frag)->segment.base.seg_len;                            \
         int* _intmem;                                                   \
         size_t align = (intptr_t)_memory & 0xFUL;                       \
         switch( align & 0x3 ) {                                         \
      * a few lines down. Note that we hand in the peer rank just for debugging
      * support. */
     rc = ep->mpool->mpool_register(ep->mpool, src_seg->memh_seg_addr.pval,
-				   src_seg->memh_seg_len, ep->peer_smp_rank,
+                                   src_seg->memh_seg_len, ep->peer_smp_rank,
                                    (mca_mpool_base_registration_t **)&reg_ptr);
 
     if (OMPI_SUCCESS != rc) {
     mca_common_wait_stream_synchronize(&rget_reg);
 
     rc = mca_common_cuda_memcpy(dst_seg->base.seg_addr.pval, remote_memory_address,
-				dst_seg->base.seg_len, "mca_btl_smcuda_get",
+                                dst_seg->base.seg_len, "mca_btl_smcuda_get",
                                 (mca_btl_base_descriptor_t *)frag, &done);
     if (OMPI_SUCCESS != rc) {
         /* Out of resources can be handled by upper layers. */

ompi/mca/btl/tcp/btl_tcp.c

         0, /* latency */
         0, /* bandwidth */
         0, /* flags */
+        0, /* segment size */
         mca_btl_tcp_add_procs,
         mca_btl_tcp_del_procs,
         NULL, 

ompi/mca/btl/udapl/btl_udapl.c

         0, /* latency */
         0, /* bandwidth */
         MCA_BTL_FLAGS_SEND,
+        0, /* segment size */
         mca_btl_udapl_add_procs,
         mca_btl_udapl_del_procs,
         NULL, 
         return NULL;
     }
 
-    frag->segment.seg_len = size;
+    frag->segment.base.seg_len = size;
 
     /* Set up the LMR triplet from the frag segment.
      * Note: The triplet.segment_len is set to what is required for
      * triplet.segment_len will have to change.
      */
     frag->triplet.virtual_address =
-        (DAT_VADDR)(uintptr_t)frag->segment.seg_addr.pval;
+        (DAT_VADDR)(uintptr_t)frag->segment.base.seg_addr.pval;
     frag->triplet.segment_length =
-        frag->segment.seg_len + sizeof(mca_btl_udapl_footer_t);
+        frag->segment.base.seg_len + sizeof(mca_btl_udapl_footer_t);
     assert(frag->triplet.lmr_context ==
         frag->registration->lmr_triplet.lmr_context);
     
                 frag->registration = (mca_btl_udapl_reg_t*)registration;
             }
 
-            frag->segment.seg_len = max_data;
-            frag->segment.seg_addr.pval = iov.iov_base;
+            frag->segment.base.seg_len = max_data;
+            frag->segment.base.seg_addr.pval = iov.iov_base;
             frag->triplet.segment_length = max_data;
             frag->triplet.virtual_address = (DAT_VADDR)(uintptr_t)iov.iov_base;
             frag->triplet.lmr_context =
     }
     
     iov.iov_len = max_data;
-    iov.iov_base = (char *) frag->segment.seg_addr.pval + reserve;
+    iov.iov_base = (char *) frag->segment.base.seg_addr.pval + reserve;
     
     rc = opal_convertor_pack(convertor,
         &iov, &iov_count, &max_data );
     *size = max_data;
 
     /* setup lengths and addresses to send out data */
-    frag->segment.seg_len = max_data + reserve;
+    frag->segment.base.seg_len = max_data + reserve;
     frag->triplet.segment_length =
         max_data + reserve + sizeof(mca_btl_udapl_footer_t);
     frag->triplet.virtual_address =
-        (DAT_VADDR)(uintptr_t)frag->segment.seg_addr.pval;
+        (DAT_VADDR)(uintptr_t)frag->segment.base.seg_addr.pval;
 
     /* initialize base descriptor */
     frag->base.des_src = &frag->segment;
         return NULL;
     }
 
-    frag->segment.seg_len = *size;
-    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
+    frag->segment.base.seg_len = *size;
+    opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.base.seg_addr.pval) );
 
     if(NULL == registration) {
         /* didn't get a memory registration passed in, so must
          * register the region now
          */ 
         rc = btl->btl_mpool->mpool_register(btl->btl_mpool,
-                                   frag->segment.seg_addr.pval,
-                                   frag->segment.seg_len,
+                                   frag->segment.base.seg_addr.pval,
+                                   frag->segment.base.seg_len,
                                    0,
                                    &registration);
         if(OMPI_SUCCESS != rc || NULL == registration) {
     frag->base.des_dst_cnt = 1;
     frag->base.des_flags = flags;
 
-    frag->segment.seg_key.key32[0] =
-        ((mca_btl_udapl_reg_t*)registration)->rmr_context;
+    frag->segment.context = ((mca_btl_udapl_reg_t*)registration)->rmr_context;
     
     frag->base.order = MCA_BTL_NO_ORDER;
 
 
     frag->endpoint = endpoint;
     frag->ftr = (mca_btl_udapl_footer_t *)
-        ((char *)frag->segment.seg_addr.pval + frag->segment.seg_len);
+        ((char *)frag->segment.base.seg_addr.pval + frag->segment.base.seg_len);
     frag->ftr->tag = tag;
     frag->type = MCA_BTL_UDAPL_SEND;
 
     int rc = OMPI_SUCCESS;
     
     mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
-    mca_btl_base_segment_t *dst_segment = des->des_dst;
+    mca_btl_udapl_segment_t *dst_segment = des->des_dst;
 
     frag->btl = (mca_btl_udapl_module_t *)btl;
     frag->endpoint = endpoint;
     } else {
         /* work queue tokens available, try to send  */
 
-	if(OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) {
+        if(OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], -1) < 0) {
             OPAL_THREAD_ADD32(&endpoint->endpoint_lwqe_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
-	    OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
-	    OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
-	    opal_list_append(&endpoint->endpoint_max_frags,
-		(opal_list_item_t*)frag);
-	    OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
-	    opal_progress();
-	} else {
-	    frag->triplet.segment_length = frag->segment.seg_len;
+            OPAL_THREAD_ADD32(&endpoint->endpoint_sr_tokens[BTL_UDAPL_MAX_CONNECTION], 1);
+            OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
+            opal_list_append(&endpoint->endpoint_max_frags,
+                (opal_list_item_t*)frag);
+            OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
+            opal_progress();
+        } else {
+            frag->triplet.segment_length = frag->segment.base.seg_len;
         
-	    remote_buffer.rmr_context =
-		(DAT_RMR_CONTEXT)dst_segment->seg_key.key32[0];
-	    remote_buffer.target_address =
-		(DAT_VADDR)(uintptr_t)dst_segment->seg_addr.lval;
-	    remote_buffer.segment_length = dst_segment->seg_len;
-
-	    cookie.as_ptr = frag;
-	    OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
-	    rc = dat_ep_post_rdma_write(endpoint->endpoint_max,
-		1,
-		&frag->triplet,
-		cookie,
-		&remote_buffer,	
-		DAT_COMPLETION_DEFAULT_FLAG);
-	    OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
-	    if(DAT_SUCCESS != rc) {
-		char* major;
-		char* minor;
-
-		dat_strerror(rc, (const char**)&major,
-		    (const char**)&minor);
-		BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_rdma_write",
-		    major, minor));
-		rc = OMPI_ERROR;
-	    }
-	}
+            remote_buffer.rmr_context = dst_segment->context;
+            remote_buffer.target_address =
+                (DAT_VADDR)(uintptr_t)dst_segment->base.seg_addr.lval;
+            remote_buffer.segment_length = dst_segment->base.seg_len;
+
+            cookie.as_ptr = frag;
+            OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
+            rc = dat_ep_post_rdma_write(endpoint->endpoint_max,
+                1,
+                &frag->triplet,
+                cookie,
+                &remote_buffer,        
+                DAT_COMPLETION_DEFAULT_FLAG);
+            OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
+            if(DAT_SUCCESS != rc) {
+                char* major;
+                char* minor;
+
+                dat_strerror(rc, (const char**)&major,
+                    (const char**)&minor);
+                BTL_ERROR(("ERROR: %s %s %s\n", "dat_ep_post_rdma_write",
+                    major, minor));
+                rc = OMPI_ERROR;
+            }
+        }
     }
     
     return rc;

ompi/mca/btl/udapl/btl_udapl_component.c

         sizeof(mca_btl_udapl_frag_eager_rdma_t) +
         mca_btl_udapl_component.udapl_eager_frag_size;
 
+    mca_btl_udapl_module.super.btl_seg_size = sizeof (mca_btl_udapl_segment_t);
+
     return rc;
 }
 
     mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)descriptor;
     mca_btl_udapl_endpoint_t* endpoint = frag->endpoint;
     mca_btl_udapl_control_header_t* ctl_hdr =
-        frag->segment.seg_addr.pval;
+        frag->segment.base.seg_addr.pval;
     
     switch (ctl_hdr->type) {
     case MCA_BTL_UDAPL_CONTROL_RDMA_CONNECT:
     {        
         mca_btl_udapl_eager_rdma_connect_t* rdma_connect =
-            frag->segment.seg_addr.pval;
+            frag->segment.base.seg_addr.pval;
 
         if (endpoint->endpoint_eager_rdma_remote.base.pval) {
             BTL_ERROR(("ERROR: Received RDMA connect twice!"));
     case MCA_BTL_UDAPL_CONTROL_RDMA_CREDIT:
     {
         mca_btl_udapl_eager_rdma_credit_t* rdma_credit =
-            frag->segment.seg_addr.pval;
+            frag->segment.base.seg_addr.pval;
         
         /* don't return credits used for rdma credit control message */
         OPAL_THREAD_ADD32(
     case MCA_BTL_UDAPL_CONTROL_SR_CREDIT:
     {
         mca_btl_udapl_sr_credit_t* sr_credit =
-            frag->segment.seg_addr.pval;
+            frag->segment.base.seg_addr.pval;
         
         /* don't return credits used for sr credit control message */
         OPAL_THREAD_ADD32(
                             flags);
     cookie.as_ptr = frag;
 
-    memcpy(frag->segment.seg_addr.pval,
+    memcpy(frag->segment.base.seg_addr.pval,
             &btl->udapl_addr, sizeof(mca_btl_udapl_addr_t));
-    memcpy((char *)frag->segment.seg_addr.pval + sizeof(mca_btl_udapl_addr_t),
+    memcpy((char *)frag->segment.base.seg_addr.pval + sizeof(mca_btl_udapl_addr_t),
             &connection_seq, sizeof(int32_t));
     connection_seq++;
 
                     assert(frag->base.des_src_cnt == 0);
                     assert(frag->type == MCA_BTL_UDAPL_RECV);
                     assert(frag->triplet.virtual_address ==
-                            (DAT_VADDR)(uintptr_t)frag->segment.seg_addr.pval);
+                            (DAT_VADDR)(uintptr_t)frag->segment.base.seg_addr.pval);
                     assert(frag->triplet.segment_length == frag->size);
                     assert(frag->btl == btl);
 
                     /* setup frag ftr location and do callback */
-                    frag->segment.seg_len = dto->transfered_length -
+                    frag->segment.base.seg_len = dto->transfered_length -
                         sizeof(mca_btl_udapl_footer_t);
                     frag->ftr = (mca_btl_udapl_footer_t *)
                         ((char *)frag->segment.seg_addr.pval + 

ompi/mca/btl/udapl/btl_udapl_frag.c

 #endif
 
     frag->registration = reg; 
-    frag->segment.seg_addr.pval = (unsigned char*)frag->base.super.ptr;
+    frag->segment.base.seg_addr.pval = (unsigned char*)frag->base.super.ptr;
     frag->ftr = NULL;
 
     /* Don't understand why yet, but there are cases where reg is NULL -
 
 static void mca_btl_udapl_frag_eager_constructor(mca_btl_udapl_frag_t* frag) 
 { 
-    frag->segment.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
+    frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
     frag->size = mca_btl_udapl_component.udapl_eager_frag_size;
     mca_btl_udapl_frag_common_constructor(frag); 
 }
 
 static void mca_btl_udapl_frag_max_constructor(mca_btl_udapl_frag_t* frag) 
 { 
-    frag->segment.seg_len = mca_btl_udapl_module.super.btl_max_send_size;
+    frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_max_send_size;
     frag->size = mca_btl_udapl_component.udapl_max_frag_size;
     mca_btl_udapl_frag_common_constructor(frag); 
 }
 static void mca_btl_udapl_frag_user_constructor(mca_btl_udapl_frag_t* frag) 
 { 
     mca_btl_udapl_frag_common_constructor(frag); 
-    frag->segment.seg_len = 0;
-    frag->segment.seg_addr.pval = NULL;
+    frag->segment.base.seg_len = 0;
+    frag->segment.base.seg_addr.pval = NULL;
     frag->ftr = NULL;
     frag->size = 0;
     frag->registration = NULL;
 static void mca_btl_udapl_frag_eager_rdma_constructor(mca_btl_udapl_frag_t* frag) 
 { 
     mca_btl_udapl_frag_eager_constructor(frag);
-    frag->segment.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
+    frag->segment.base.seg_len = mca_btl_udapl_module.super.btl_eager_limit;
     frag->size = mca_btl_udapl_component.udapl_eager_frag_size;
     frag->rdma_ftr = (mca_btl_udapl_rdma_footer_t *)
-        ((char *)(frag->segment.seg_addr.pval) +
+        ((char *)(frag->segment.base.seg_addr.pval) +
                 frag->size -
                 sizeof(mca_btl_udapl_rdma_footer_t));
     frag->rdma_ftr->active=0;
     frag->ftr = NULL;
     frag->size = 0; 
     frag->registration = NULL; 
-    frag->segment.seg_len = 0;
-    frag->segment.seg_addr.pval = NULL; 
+    frag->segment.base.seg_len = 0;
+    frag->segment.base.seg_addr.pval = NULL; 
     
     frag->base.des_src = NULL;
     frag->base.des_src_cnt = 0;

ompi/mca/btl/udapl/btl_udapl_frag.h

 };
 typedef struct mca_btl_udapl_rdma_footer_t mca_btl_udapl_rdma_footer_t;
 
+struct mca_btl_udapl_segment_t {
+    mca_btl_base_segment_t base;
+    DAT_RMR_CONTEXT context;
+};
+typedef struct mca_btl_udapl_segment_t mca_btl_udapl_segment_t;
+
 /**
  * uDAPL fragment derived type.
  */
 struct mca_btl_udapl_frag_t {
     mca_btl_base_descriptor_t base;
-    mca_btl_base_segment_t segment;
+    mca_btl_udapl_segment_t segment;
 
     struct mca_btl_udapl_module_t* btl;
     struct mca_btl_base_endpoint_t* endpoint; 

ompi/mca/btl/ugni/btl_ugni_component.c

     mca_btl_ugni_component.ugni_get_limit =
         mca_btl_ugni_param_register_int("get_limit", "Maximum size message that "
                                         "will be sent using a get protocol "
-                                        "(default 4M)", 4 * 1024 * 1024);
+                                        "(default 1M)", 1 * 1024 * 1024);
 
     mca_btl_ugni_component.rdma_max_retries =
         mca_btl_ugni_param_register_int("rdma_max_retries", NULL, 16);
         return 0;
     }
 
-    BTL_VERBOSE(("RDMA/FMA complete for frag %p", frag));
+    BTL_VERBOSE(("RDMA/FMA complete for frag %p", (void *) frag));
 
     frag->cbfunc (frag, ompi_common_rc_ugni_to_ompi (rc));
 
             (mca_btl_base_endpoint_t *) opal_list_remove_first (&ugni_module->ep_wait_list);
         assert (NULL != endpoint);
 
+        endpoint->wait_listed = false;
+
         rc = mca_btl_progress_send_wait_list (endpoint);
-        if (OMPI_SUCCESS != rc) {
+        if (OMPI_SUCCESS != rc && false == endpoint->wait_listed) {
             opal_list_append (&ugni_module->ep_wait_list, &endpoint->super);
+            endpoint->wait_listed = true;
         }
     }
 

ompi/mca/btl/ugni/btl_ugni_endpoint.h

     struct mca_btl_ugni_smsg_mbox_t *mailbox;
 
     opal_list_t frag_wait_list;
+    bool wait_listed;
 
     int32_t smsg_progressing;
 } mca_btl_base_endpoint_t;

ompi/mca/btl/ugni/btl_ugni_frag.h

 typedef struct mca_btl_ugni_segment_t {
     mca_btl_base_segment_t base;
     gni_mem_handle_t       memory_handle;
+    uint8_t                extra_bytes[3];
+    uint8_t                extra_byte_count;
 } mca_btl_ugni_segment_t;
 
 typedef struct mca_btl_ugni_base_frag_t {
     mca_btl_base_descriptor_t    base;
     size_t                       hdr_size;
     mca_btl_ugni_frag_hdr_t      hdr;
-    mca_btl_ugni_segment_t       rdma_segments[2];
-    mca_btl_base_segment_t       segments[2];
+    mca_btl_ugni_segment_t       segments[2];
     ompi_common_ugni_post_desc_t post_desc;
     mca_btl_base_endpoint_t     *endpoint;
     mca_btl_ugni_reg_t          *registration;

ompi/mca/btl/ugni/btl_ugni_get.c

                       struct mca_btl_base_endpoint_t *endpoint,
                       struct mca_btl_base_descriptor_t *des) {
     mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) des;
-    size_t size = des->des_src->seg_len;
+    mca_btl_ugni_segment_t *src_seg = (mca_btl_ugni_segment_t *) des->des_src;
+    mca_btl_ugni_segment_t *dst_seg = (mca_btl_ugni_segment_t *) des->des_dst;
+    size_t size = src_seg->base.seg_len - src_seg->extra_byte_count;
     bool check;
 
     BTL_VERBOSE(("Using RDMA/FMA Get"));
         return OMPI_ERR_NOT_AVAILABLE;
     }
 
+    if (src_seg->extra_byte_count) {
+        memmove (dst_seg->base.seg_addr.pval + size, src_seg->extra_bytes, src_seg->extra_byte_count);
+        src_seg->base.seg_len = size;
+        dst_seg->base.seg_len = size;
+    }
+
     des->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
 
-    return mca_btl_ugni_post (frag, true, (mca_btl_ugni_segment_t *) des->des_dst,
-                              (mca_btl_ugni_segment_t *) des->des_src);
+    return mca_btl_ugni_post (frag, true, dst_seg, src_seg);
 }
 
 static void mca_btl_ugni_frag_set_ownership (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
 
         segs[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
         segs[0].seg_len       = hdr_len;
-        segs[1].seg_addr.pval = frag->segments[0].seg_addr.pval;
+        segs[1].seg_addr.pval = frag->segments[0].base.seg_addr.pval;
         segs[1].seg_len       = payload_len;
     } else {
         tmp.base.des_dst_cnt = 1;
 
-        segs[0].seg_addr.pval = frag->segments[0].seg_addr.pval;
+        segs[0].seg_addr.pval = frag->segments[0].base.seg_addr.pval;
         segs[0].seg_len       = payload_len;
     }
 
         frag->base.des_cbfunc = mca_btl_ugni_callback_eager_get;
         frag->base.des_flags  = MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
 
-        frag->base.des_dst = frag->segments;
-        frag->base.des_dst_cnt = 1;
-
-        frag->segments[1] = hdr.eager.src_seg;
-        frag->base.des_src = frag->segments + 1;
-        frag->base.des_src_cnt = 1;
+        frag->segments[1].base = hdr.eager.src_seg;
 
         /* increase size to a multiple of 4 bytes (required for get) */
         frag->segments[0].base.seg_len = frag->segments[1].base.seg_len =
             (hdr.eager.src_seg.seg_len + 3) & ~3;
 
-        rc = mca_btl_ugni_post (frag, true, frag->base.des_dst, frag->base.des_src);
+        rc = mca_btl_ugni_post (frag, true, frag->segments, frag->segments + 1);
         if (OPAL_UNLIKELY(OMPI_SUCCESS == rc)) {
             return OMPI_SUCCESS;
         }

ompi/mca/btl/ugni/btl_ugni_module.c

         /* .btl_latency = */                   0,
         /* .btl_bandwidth = */                 0,
         /* .btl_flags = */                     0,
+        /* .btl_seg_size = */                  0,
 
         /* member functions */
         mca_btl_ugni_add_procs,
 
     frag->base.des_flags = flags;
     frag->base.order = order;
-    frag->base.des_src = frag->segments + 1;
+    frag->base.des_src = &frag->segments[1].base;
     frag->base.des_src_cnt = 1;
-    frag->base.des_dst = frag->segments + 1;
+    frag->base.des_dst = &frag->segments[1].base;
     frag->base.des_dst_cnt = 1;
 
     frag->segments[0].base.seg_addr.pval = NULL;
     frag->segments[0].base.seg_len       = *size;
     frag->segments[0].base.seg_addr.pval = data_ptr;
 
-    frag->base.des_dst     = frag->segments;
+    frag->base.des_dst     = &frag->segments->base;
     frag->base.des_dst_cnt = 1;
     frag->base.order       = order;
     frag->base.des_flags   = flags;

ompi/mca/btl/ugni/btl_ugni_prepare.h

 
 static inline struct mca_btl_base_descriptor_t *
 mca_btl_ugni_prepare_src_send_inplace (struct mca_btl_base_module_t *btl,
-				       mca_btl_base_endpoint_t *endpoint,
-				       struct opal_convertor_t *convertor,
-				       uint8_t order, size_t reserve, size_t *size,
-				       uint32_t flags)
+                                       mca_btl_base_endpoint_t *endpoint,
+                                       struct opal_convertor_t *convertor,
+                                       uint8_t order, size_t reserve, size_t *size,
+                                       uint32_t flags)
 {
     bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
     mca_btl_ugni_base_frag_t *frag = NULL;
     frag->segments[1].base.seg_addr.pval = data_ptr;
     frag->segments[1].base.seg_len       = *size;
 
-    frag->base.des_src     = frag->segments;
+    frag->base.des_src     = &frag->segments->base;
     frag->base.des_src_cnt = 2;
     frag->base.order       = order;
     frag->base.des_flags   = flags;
     frag->segments[1].base.seg_addr.pval = frag->base.super.ptr;
     frag->segments[1].base.seg_len       = *size;
 
-    frag->base.des_src     = frag->segments;
+    frag->base.des_src     = &frag->segments->base;
     frag->base.des_src_cnt = 2;
     frag->base.order       = order;
     frag->base.des_flags   = flags;
 
 static inline struct mca_btl_base_descriptor_t *
 mca_btl_ugni_prepare_src_send (struct mca_btl_base_module_t *btl,
-			       mca_btl_base_endpoint_t *endpoint,
-			       struct opal_convertor_t *convertor,
-			       uint8_t order, size_t reserve, size_t *size,
-			       uint32_t flags)
+                               mca_btl_base_endpoint_t *endpoint,
+                               struct opal_convertor_t *convertor,
+                               uint8_t order, size_t reserve, size_t *size,
+                               uint32_t flags)
 {
     bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
     bool send_in_place;
 
 static inline struct mca_btl_base_descriptor_t *
 mca_btl_ugni_prepare_src_rdma (struct mca_btl_base_module_t *btl,
-			       mca_btl_base_endpoint_t *endpoint,
+                               mca_btl_base_endpoint_t *endpoint,
                                mca_mpool_base_registration_t *registration,
-			       struct opal_convertor_t *convertor,
-			       uint8_t order, size_t *size,
-			       uint32_t flags)
+                               struct opal_convertor_t *convertor,
+                               uint8_t order, size_t *size,
+                               uint32_t flags)
 {
     mca_btl_ugni_base_frag_t *frag;
     void *data_ptr;
                 sizeof (frag->segments[0].memory_handle));
     }
 
+    if ((flags & MCA_BTL_DES_FLAGS_GET) && (*size & 0x3)) {
+        memmove (frag->segments[0].extra_bytes, (uintptr_t) data_ptr + (*size & ~0x3),
+                 *size & 0x3);
+        frag->segments[0].extra_byte_count = *size & 0x3;
+    } else {
+        frag->segments[0].extra_byte_count = 0;
+    }
+
     frag->segments[0].base.seg_addr.pval = data_ptr;
     frag->segments[0].base.seg_len       = *size;
 
-    frag->base.des_src     = frag->segments;
+    frag->base.des_src     = &frag->segments->base;
     frag->base.des_src_cnt = 1;
     frag->base.order       = order;
     frag->base.des_flags   = flags;

ompi/mca/btl/ugni/btl_ugni_send.c

                        mca_btl_base_tag_t tag)
 {
     mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) descriptor;
-    size_t size = frag->segments[0].seg_len + frag->segments[1].seg_len;
+    size_t size = frag->segments[0].base.seg_len + frag->segments[1].base.seg_len;
     mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
     int flags_save = frag->base.des_flags;
     int rc;
 
-    BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %d", (void *)descriptor,
-                 ORTE_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, frag->segments[0].seg_len));
+    BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %" PRIu64, (void *)descriptor,
+                 ORTE_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, frag->segments[0].base.seg_len));
 
     /* tag and len are at the same location in eager and smsg frag hdrs */
     frag->hdr.send.lag = (tag << 24) | size;
 
     if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc)) {
         /* queue up request */
-        if (0 == opal_list_get_size (&endpoint->frag_wait_list)) {
+        if (false == endpoint->wait_listed) {
             opal_list_append (&ugni_module->ep_wait_list, &endpoint->super);
+            endpoint->wait_listed = true;
         }
 
         opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag);

ompi/mca/btl/ugni/btl_ugni_smsg.c

     gni_cq_entry_t event_data;
     gni_return_t grc;
     uint64_t inst_id;
-    int rc;
 
     grc = GNI_CqGetEvent (btl->smsg_remote_cq, &event_data);
     if (GNI_RC_NOT_DONE == grc) {

ompi/mca/btl/ugni/btl_ugni_smsg.h

                                             MCA_BTL_UGNI_TAG_SEND);
     }
 
-    frag->hdr.eager.src_seg = frag->segments[1];
+    frag->hdr.eager.src_seg = frag->segments[1].base;
     frag->hdr.eager.ctx     = (void *) frag;
 
     return ompi_mca_btl_ugni_smsg_send (frag, &frag->hdr.eager, frag->hdr_size,

ompi/mca/btl/vader/btl_vader.c

         0, /* bTl_latency */
         0, /* btl_bandwidth */
         0, /* btl_flags */
+        0, /* btl segment size */
         vader_add_procs,
         vader_del_procs,
         NULL, /* btl_register */

ompi/mca/pml/bfo/pml_bfo_hdr.h

     uint8_t hdr_padding[4];
 #endif
     ompi_ptr_t hdr_des;                       /**< source descriptor */
-    mca_btl_base_segment_t hdr_segs[1];       /**< list of segments for rdma */
 };
 typedef struct mca_pml_bfo_rget_hdr_t mca_pml_bfo_rget_hdr_t;
 

ompi/mca/pml/bfo/pml_bfo_recvreq.c

         dst->des_cbfunc = mca_pml_bfo_put_completion;
         dst->des_cbdata = recvreq;
 
-	seg_size = btl->btl_seg_size * dst->des_dst_cnt
+	seg_size = btl->btl_seg_size * dst->des_dst_cnt;
 
         /* prepare a descriptor for rdma control message */
-        mca_bml_base_alloc(bml_btl, &ctl, MCA_BTL_NO_ORDER, sizeof(mca_pml_ob1_rdma_hdr_t) + seg_size,
+        mca_bml_base_alloc(bml_btl, &ctl, MCA_BTL_NO_ORDER, sizeof(mca_pml_bfo_rdma_hdr_t) + seg_size,
                            MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
 
         if( OPAL_UNLIKELY(NULL == ctl) ) {

ompi/mca/pml/csum/pml_csum_hdr.h

     uint8_t hdr_padding[4];
 #endif
     ompi_ptr_t hdr_des;                       /**< source descriptor */
-    mca_btl_base_segment_t hdr_segs[1];       /**< list of segments for rdma */
 };
 typedef struct mca_pml_csum_rget_hdr_t mca_pml_csum_rget_hdr_t;
 

ompi/mca/pml/csum/pml_csum_rdmafrag.h

     mca_pml_csum_hdr_t rdma_hdr;
     mca_pml_csum_rdma_state_t rdma_state;
     size_t rdma_length;
-    mca_btl_base_segment_t rdma_segs[MCA_BTL_DES_MAX_SEGMENTS];
+    uint8_t rdma_segs[MCA_BTL_SEG_MAX_SIZE * MCA_BTL_DES_MAX_SEGMENTS];
     void *rdma_req;
     struct mca_bml_base_endpoint_t* rdma_ep;
     opal_convertor_t convertor;

ompi/mca/pml/csum/pml_csum_recvreq.c

     bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_bml; 
 
     /* allocate/initialize a fragment */
+    memmove (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt);
+
     for(i = 0; i < hdr->hdr_seg_cnt; i++) {
-        frag->rdma_segs[i] = hdr->hdr_segs[i];
+        mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *)(frag->rdma_segs + i * btl->btl_seg_size);
+
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
         if ((recvreq->req_recv.req_base.req_proc->proc_arch & OPAL_ARCH_ISBIGENDIAN) !=
             (ompi_proc_local()->proc_arch & OPAL_ARCH_ISBIGENDIAN)) {
-            size += opal_swap_bytes4(hdr->hdr_segs[i].seg_len);
+            size += opal_swap_bytes4(seg->seg_len);
         } else 
 #endif
         {
-            size += hdr->hdr_segs[i].seg_len;
+            size += seg->seg_len;
         }
     }
     frag->rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl);
 
     while(bytes_remaining > 0 &&
            recvreq->req_pipeline_depth < mca_pml_csum.recv_pipeline_depth) {
-        size_t hdr_size;
-        size_t size;
+        size_t size, seg_size;
         mca_pml_csum_rdma_hdr_t* hdr;
         mca_btl_base_descriptor_t* dst;
         mca_btl_base_descriptor_t* ctl;
         dst->des_cbfunc = mca_pml_csum_put_completion;
         dst->des_cbdata = recvreq;
 
-        /* prepare a descriptor for rdma control message */
-        hdr_size = sizeof(mca_pml_csum_rdma_hdr_t);
-        if(dst->des_dst_cnt > 1) {
-            hdr_size += (sizeof(mca_btl_base_segment_t) *
-                    (dst->des_dst_cnt-1));
-        }
+        seg_size = btl->btl_seg_size * dst->des_dst_cnt;
 
-        mca_bml_base_alloc(bml_btl, &ctl, MCA_BTL_NO_ORDER, hdr_size,
+        /* prepare a descriptor for rdma control message */
+        mca_bml_base_alloc(bml_btl, &ctl, MCA_BTL_NO_ORDER, sizeof(mca_pml_csum_rdma_hdr_t) + seg_size,
                            MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP | MCA_BTL_DES_SEND_ALWAYS_CALLBACK);
 
         if( OPAL_UNLIKELY(NULL == ctl) ) {
         hdr->hdr_rdma_offset = recvreq->req_rdma_offset;
         hdr->hdr_seg_cnt = dst->des_dst_cnt;
 
-        for( i = 0; i < dst->des_dst_cnt; i++ ) {
-            hdr->hdr_segs[i].seg_addr.lval = ompi_ptr_ptol(dst->des_dst[i].seg_addr.pval);
-            hdr->hdr_segs[i].seg_len       = dst->des_dst[i].seg_len;
-            hdr->hdr_segs[i].seg_key.key64[0] = dst->des_dst[i].seg_key.key64[0];
-            hdr->hdr_segs[i].seg_key.key64[1] = dst->des_dst[i].seg_key.key64[1];
-        }
+        /* copy segments */
+        memmove (hdr + 1, dst->des_dst, seg_size);
 
         if(!recvreq->req_ack_sent)
             recvreq->req_ack_sent = true;

ompi/mca/pml/csum/pml_csum_sendreq.c

      * operation is achieved.
      */
 
-    mca_btl_base_descriptor_t* des;
+    mca_btl_base_descriptor_t* des, *src = NULL;
     mca_btl_base_segment_t* segment;
     mca_pml_csum_hdr_t* hdr;
     bool need_local_cb = false;
     if((sendreq->req_rdma_cnt == 1) && (bml_btl->btl_flags & MCA_BTL_FLAGS_GET)) {
         mca_mpool_base_registration_t* reg = sendreq->req_rdma[0].btl_reg;
         mca_btl_base_descriptor_t* src;
-        size_t i;
+        size_t i, seg_size;
         size_t old_position = sendreq->req_send.req_base.req_convertor.bConverted;
 
         MEMCHECKER(
         src->des_cbfunc = mca_pml_csum_rget_completion;
         src->des_cbdata = sendreq;
 
+        seg_size = bml_btl->btl->btl_seg_size * src->des_src_cnt;
+
         /* allocate space for get hdr + segment list */
         mca_bml_base_alloc(bml_btl, &des, MCA_BTL_NO_ORDER,
-                           sizeof(mca_pml_csum_rget_hdr_t) +
-                           (sizeof(mca_btl_base_segment_t) * (src->des_src_cnt-1)),
+                           sizeof(mca_pml_csum_rget_hdr_t) + seg_size,
                            MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
         if( OPAL_UNLIKELY(NULL == des) ) {
             opal_convertor_set_position( &sendreq->req_send.req_base.req_convertor,
         csum_hdr_hton(hdr, MCA_PML_CSUM_HDR_TYPE_RGET,
                      sendreq->req_send.req_base.req_proc);
 
-        for( i = 0; i < src->des_src_cnt; i++ ) {
-            hdr->hdr_rget.hdr_segs[i].seg_addr.lval = ompi_ptr_ptol(src->des_src[i].seg_addr.pval);
-            hdr->hdr_rget.hdr_segs[i].seg_len       = src->des_src[i].seg_len;
-            hdr->hdr_rget.hdr_segs[i].seg_key.key64[0] = src->des_src[i].seg_key.key64[0];
-            hdr->hdr_rget.hdr_segs[i].seg_key.key64[1] = src->des_src[i].seg_key.key64[1];
-        }
+        /* copy segment data */
+        memmove (hdr + 1, src->des_src, seg_size);
 
         des->des_cbfunc = mca_pml_csum_send_ctl_completion;
 
         return OMPI_SUCCESS;
     }
     mca_bml_base_free(bml_btl, des);
+    if (src) {
+        mca_bml_base_free (bml_btl, src);
+    }
+
     return rc;
 }
 
     }
 
     /* setup fragment */
+    memmove (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt);
+
     for( i = 0; i < hdr->hdr_seg_cnt; i++ ) {
-        frag->rdma_segs[i].seg_addr.lval = hdr->hdr_segs[i].seg_addr.lval;
-        frag->rdma_segs[i].seg_len       = hdr->hdr_segs[i].seg_len;
-        frag->rdma_segs[i].seg_key.key64[0] = hdr->hdr_segs[i].seg_key.key64[0];
-        frag->rdma_segs[i].seg_key.key64[1] = hdr->hdr_segs[i].seg_key.key64[1];
+        mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *) ((uintptr_t)(frag->rdma_segs) + i * btl->btl_seg_size);
 
 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
         if ((sendreq->req_send.req_base.req_proc->proc_arch & OPAL_ARCH_ISBIGENDIAN) !=
             (ompi_proc_local()->proc_arch & OPAL_ARCH_ISBIGENDIAN)) {
-            size += opal_swap_bytes4(frag->rdma_segs[i].seg_len);
+            size += opal_swap_bytes4(seg->seg_len);
         } else 
 #endif
         {
-            size += frag->rdma_segs[i].seg_len;
+            size += seg->seg_len;
         }
     }
 

ompi/mca/pml/ob1/pml_ob1_hdr.h

     uint8_t hdr_padding[4];
 #endif
     ompi_ptr_t hdr_des;                       /**< source descriptor */
-    mca_btl_base_segment_t hdr_segs[1];       /**< list of segments for rdma */
 };
 typedef struct mca_pml_ob1_rget_hdr_t mca_pml_ob1_rget_hdr_t;
 

ompi/mca/pml/ob1/pml_ob1_recvreq.c

     hdr->hdr_seg_cnt = dst->des_dst_cnt;
 
     /* copy segments */
-    memmove (hdr + 1, dst->des_dst, seg_size);
+    memcpy (hdr + 1, dst->des_dst, seg_size);
 
     dst->des_cbfunc = mca_pml_ob1_put_completion;
     dst->des_cbdata = recvreq;
     bml_endpoint = (mca_bml_base_endpoint_t*)recvreq->req_recv.req_base.req_proc->proc_bml; 
 
     /* allocate/initialize a fragment */
-    memmove (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt);
+    memcpy (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt);
 
     for(i = 0; i < hdr->hdr_seg_cnt; i++) {
         mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *)(frag->rdma_segs + i * btl->btl_seg_size);

ompi/mca/pml/ob1/pml_ob1_sendreq.c

                      sendreq->req_send.req_base.req_proc);
 
         /* copy segment data */
-        memmove (hdr + 1, src->des_src, seg_size);
+        memcpy (hdr + 1, src->des_src, seg_size);
 
         des->des_cbfunc = mca_pml_ob1_send_ctl_completion;
 
     }
 
     /* setup fragment */
-    memmove (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt);
+    memcpy (frag->rdma_segs, hdr + 1, btl->btl_seg_size * hdr->hdr_seg_cnt);
 
     for( i = 0; i < hdr->hdr_seg_cnt; i++ ) {
         mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *) ((uintptr_t)(frag->rdma_segs) + i * btl->btl_seg_size);