Source

mpi3-fortran / ompi / mca / btl / openib / btl_openib_frag.h

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2006 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2009      IBM Corporation.  All rights reserved.
 * Copyright (c) 2006-2009 Los Alamos National Security, LLC.  All rights
 *                         reserved.
 * Copyright (c) 2006-2007 Voltaire All rights reserved.
 * Copyright (c) 2010-2012 Oracle and/or its affiliates.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */

#ifndef MCA_BTL_IB_FRAG_H
#define MCA_BTL_IB_FRAG_H

#include "ompi_config.h"
#include "opal/align.h"

#include <infiniband/verbs.h>
#include "ompi/mca/btl/btl.h"

BEGIN_C_DECLS

struct mca_btl_openib_reg_t;

struct mca_btl_openib_header_t {
    mca_btl_base_tag_t tag;
    uint8_t cm_seen;
    uint16_t credits;
#if OMPI_OPENIB_PAD_HDR 
    uint8_t padding[4];
#endif
};
typedef struct mca_btl_openib_header_t mca_btl_openib_header_t;
#define BTL_OPENIB_RDMA_CREDITS_FLAG (1<<15)
#define BTL_OPENIB_IS_RDMA_CREDITS(I) ((I)&BTL_OPENIB_RDMA_CREDITS_FLAG)
#define BTL_OPENIB_CREDITS(I) ((I)&~BTL_OPENIB_RDMA_CREDITS_FLAG)

#define BTL_OPENIB_HEADER_HTON(h)     \
do {                                  \
    (h).credits = htons((h).credits); \
} while (0)

#define BTL_OPENIB_HEADER_NTOH(h)     \
do {                                  \
    (h).credits = ntohs((h).credits); \
} while (0)

typedef struct mca_btl_openib_header_coalesced_t {
    mca_btl_base_tag_t tag;
    uint32_t size;
    uint32_t alloc_size;
#if OMPI_OPENIB_PAD_HDR
    uint8_t padding[4];
#endif
} mca_btl_openib_header_coalesced_t;

#define BTL_OPENIB_HEADER_COALESCED_NTOH(h)     \
    do {                                        \
        (h).size = ntohl((h).size);             \
        (h).alloc_size = ntohl((h).alloc_size); \
     } while(0)

#define BTL_OPENIB_HEADER_COALESCED_HTON(h)     \
    do {                                        \
        (h).size = htonl((h).size);             \
        (h).alloc_size = htonl((h).alloc_size); \
     } while(0)

#if OMPI_OPENIB_PAD_HDR
/* BTL_OPENIB_FTR_PADDING
 * This macro is used to keep the pointer to openib footers aligned for 
 * systems like SPARC64 that take a big performance hit when addresses 
 * are not aligned (and by default sigbus instead of coercing the type on 
 * an unaligned address).
 *
 * We assure alignment of a packet's structures when OMPI_OPENIB_PAD_HDR 
 * is set to 1.  When this is the case then several structures are padded
 * to assure alignment and the mca_btl_openib_footer_t structure itself
 * will uses the BTL_OPENIB_FTR_PADDING macro to shift the location of the 
 * pointer to assure proper alignment after the PML Header and data.
 * For example sending a 1 byte data packet the memory layout without 
 * footer alignment would look something like the following:
 * 
 * 0x00   : mca_btl_openib_coalesced_header_t (12 bytes + 4 byte pad)
 * 0x10   : mca_btl_openib_control_header_t (1 byte + 7 byte pad)
 * 0x18   : mca_btl_openib_header_t (4 bytes + 4 byte pad)
 * 0x20   : PML Header and data (16 bytes PML + 1 byte data)
 * 0x29   : mca_btl_openib_footer_t (4 bytes + 4 byte pad)
 * 0x31   : end of packet
 *
 * By applying the BTL_OPENIB_FTR_PADDING() in the progress_one_device
 * and post_send routines we adjust the pointer to mca_btl_openib_footer_t 
 * from 0x29 to 0x2C thus correctly aligning the start of the 
 * footer pointer.  This adjustment will cause the padding field of  
 * mca_btl_openib_footer_t to overlap with the neighboring memory but since
 * we never use the padding we do not end up inadvertently overwriting
 * memory that does not belong to the fragment.
 */
#define BTL_OPENIB_FTR_PADDING(size) \
    OPAL_ALIGN_PAD_AMOUNT(size, sizeof(uint64_t))

/* BTL_OPENIB_ALIGN_COALESCE_HDR
 * This macro is used in btl_openib.c, while creating a coalesce fragment, 
 * to align the coalesce headers.
 */
#define BTL_OPENIB_ALIGN_COALESCE_HDR(ptr) \
  OPAL_ALIGN_PTR(ptr, sizeof(uint32_t), unsigned char*)

/* BTL_OPENIB_COALESCE_HDR_PADDING
 * This macro is used in btl_openib_component.c, while parsing an incoming 
 * coalesce fragment, to determine the padding amount used to align the 
 * mca_btl_openib_coalesce_hdr_t.
 */
#define BTL_OPENIB_COALESCE_HDR_PADDING(ptr) \
  OPAL_ALIGN_PAD_AMOUNT(ptr, sizeof(uint32_t))
#else
#define BTL_OPENIB_FTR_PADDING(size) 0
#define BTL_OPENIB_ALIGN_COALESCE_HDR(ptr) ptr
#define BTL_OPENIB_COALESCE_HDR_PADDING(ptr) 0
#endif

struct mca_btl_openib_footer_t {
#if OPAL_ENABLE_DEBUG
    uint32_t seq;
#endif
    union {
        uint32_t size;
        uint8_t buf[4];
    } u;
#if OMPI_OPENIB_PAD_HDR
#if OPAL_ENABLE_DEBUG
    /* this footer needs to be of a 8-byte multiple so by adding the
     * seq field you throw this off and you cannot just remove the 
     * padding because the padding is needed in order to adjust the alignment
     * and not overwrite other packets.
     */
    uint8_t padding[12];
#else 
    uint8_t padding[8];
#endif
#endif
};
typedef struct mca_btl_openib_footer_t mca_btl_openib_footer_t;

#ifdef WORDS_BIGENDIAN
#define MCA_BTL_OPENIB_FTR_SIZE_REVERSE(ftr)
#else
#define MCA_BTL_OPENIB_FTR_SIZE_REVERSE(ftr)    \
    do {                                        \
        uint8_t tmp = (ftr).u.buf[0];           \
        (ftr).u.buf[0]=(ftr).u.buf[2];          \
        (ftr).u.buf[2]=tmp;                     \
    } while (0)
#endif

#if OPAL_ENABLE_DEBUG
#define BTL_OPENIB_FOOTER_SEQ_HTON(h)  ((h).seq = htonl((h).seq))
#define BTL_OPENIB_FOOTER_SEQ_NTOH(h)  ((h).seq = ntohl((h).seq))
#else
#define BTL_OPENIB_FOOTER_SEQ_HTON(h)
#define BTL_OPENIB_FOOTER_SEQ_NTOH(h)
#endif

#define BTL_OPENIB_FOOTER_HTON(h)               \
    do {                                        \
        BTL_OPENIB_FOOTER_SEQ_HTON(h);          \
        MCA_BTL_OPENIB_FTR_SIZE_REVERSE(h);     \
    } while (0)

#define BTL_OPENIB_FOOTER_NTOH(h)               \
    do {                                        \
        BTL_OPENIB_FOOTER_SEQ_NTOH(h);          \
        MCA_BTL_OPENIB_FTR_SIZE_REVERSE(h);     \
    } while (0)

#define MCA_BTL_OPENIB_CONTROL_CREDITS      0
#define MCA_BTL_OPENIB_CONTROL_RDMA         1
#define MCA_BTL_OPENIB_CONTROL_COALESCED    2
#define MCA_BTL_OPENIB_CONTROL_CTS          3
#if BTL_OPENIB_FAILOVER_ENABLED
#define MCA_BTL_OPENIB_CONTROL_EP_BROKEN    4
#define MCA_BTL_OPENIB_CONTROL_EP_EAGER_RDMA_ERROR 5
#endif

struct mca_btl_openib_control_header_t {
    uint8_t  type;
#if OMPI_OPENIB_PAD_HDR
    uint8_t  padding[7];
#endif
};
typedef struct mca_btl_openib_control_header_t mca_btl_openib_control_header_t;

struct mca_btl_openib_eager_rdma_header_t {
    mca_btl_openib_control_header_t control;
    uint32_t rkey;
    ompi_ptr_t rdma_start;
};
typedef struct mca_btl_openib_eager_rdma_header_t mca_btl_openib_eager_rdma_header_t;

#define BTL_OPENIB_EAGER_RDMA_CONTROL_HEADER_HTON(h)       \
    do {                                                   \
        (h).rkey = htonl((h).rkey);                        \
        (h).rdma_start.lval = hton64((h).rdma_start.lval); \
    } while (0)

#define BTL_OPENIB_EAGER_RDMA_CONTROL_HEADER_NTOH(h)         \
    do {                                                     \
        (h).rkey = ntohl((h).rkey);                          \
        (h).rdma_start.lval = ntoh64((h).rdma_start.lval);   \
    } while (0)


struct mca_btl_openib_rdma_credits_header_t {
    mca_btl_openib_control_header_t control;
#if OMPI_OPENIB_PAD_HDR
    uint8_t  padding[1];
#endif
    uint8_t qpn;
    uint16_t rdma_credits;
};
typedef struct mca_btl_openib_rdma_credits_header_t mca_btl_openib_rdma_credits_header_t;

#define BTL_OPENIB_RDMA_CREDITS_HEADER_HTON(h)     \
do {                                               \
    (h).rdma_credits = htons((h).rdma_credits);    \
} while (0)

#define BTL_OPENIB_RDMA_CREDITS_HEADER_NTOH(h)     \
do {                                               \
    (h).rdma_credits = ntohs((h).rdma_credits);    \
} while (0)

#if BTL_OPENIB_FAILOVER_ENABLED
struct mca_btl_openib_broken_connection_header_t {
    mca_btl_openib_control_header_t control;
    uint32_t lid;
    uint64_t subnet_id;
    uint32_t vpid;
    uint32_t index; /* for eager RDMA only */
};
typedef struct mca_btl_openib_broken_connection_header_t mca_btl_openib_broken_connection_header_t;

#define BTL_OPENIB_BROKEN_CONNECTION_HEADER_HTON(h)    \
    do {                                               \
        (h).lid = htonl((h).lid);                      \
        (h).subnet_id = hton64((h).subnet_id);         \
        (h).vpid = htonl((h).vpid);                    \
        (h).index = htonl((h).index);                  \
    } while (0)

#define BTL_OPENIB_BROKEN_CONNECTION_HEADER_NTOH(h)    \
    do {                                               \
        (h).lid = ntohl((h).lid);                      \
        (h).subnet_id = ntoh64((h).subnet_id);         \
        (h).vpid = ntohl((h).vpid);                    \
        (h).index = ntohl((h).index);                  \
    } while (0)
#endif
enum mca_btl_openib_frag_type_t {
    MCA_BTL_OPENIB_FRAG_RECV,
    MCA_BTL_OPENIB_FRAG_RECV_USER,
    MCA_BTL_OPENIB_FRAG_SEND,
    MCA_BTL_OPENIB_FRAG_SEND_USER,
    MCA_BTL_OPENIB_FRAG_EAGER_RDMA,
    MCA_BTL_OPENIB_FRAG_CONTROL,
    MCA_BTL_OPENIB_FRAG_COALESCED
};
typedef enum mca_btl_openib_frag_type_t mca_btl_openib_frag_type_t;

#define openib_frag_type(f) (to_base_frag(f)->type)
/**
 * IB fragment derived type.
 */

typedef struct mca_btl_openib_segment_t {
    mca_btl_base_segment_t base;
    uint32_t key;
} mca_btl_openib_segment_t;

/* base openib frag */
typedef struct mca_btl_openib_frag_t {
    mca_btl_base_descriptor_t base;
    mca_btl_openib_segment_t segment;
    mca_btl_openib_frag_type_t type;
    ompi_free_list_t* list;
} mca_btl_openib_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_frag_t);

#define to_base_frag(f) ((mca_btl_openib_frag_t*)(f))

/* frag used for communication */
typedef struct mca_btl_openib_com_frag_t {
    mca_btl_openib_frag_t super;
    struct ibv_sge sg_entry;
    struct mca_btl_openib_reg_t *registration;
    struct mca_btl_base_endpoint_t *endpoint;
    /* number of unsignaled frags sent before this frag. */
    uint32_t n_wqes_inflight;
} mca_btl_openib_com_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_com_frag_t);

#define to_com_frag(f) ((mca_btl_openib_com_frag_t*)(f))

typedef struct mca_btl_openib_out_frag_t {
    mca_btl_openib_com_frag_t super;
    struct ibv_send_wr sr_desc;
} mca_btl_openib_out_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_out_frag_t);

#define to_out_frag(f) ((mca_btl_openib_out_frag_t*)(f))

typedef struct mca_btl_openib_com_frag_t mca_btl_openib_in_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_in_frag_t);

#define to_in_frag(f) ((mca_btl_openib_in_frag_t*)(f))

typedef struct mca_btl_openib_send_frag_t {
    mca_btl_openib_out_frag_t super;
    mca_btl_openib_header_t *hdr, *chdr;
    mca_btl_openib_footer_t *ftr;
    uint8_t qp_idx;
    uint32_t coalesced_length;
    opal_list_t coalesced_frags;
} mca_btl_openib_send_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_send_frag_t);

#define to_send_frag(f) ((mca_btl_openib_send_frag_t*)(f))

typedef struct mca_btl_openib_recv_frag_t {
    mca_btl_openib_in_frag_t super;
    mca_btl_openib_header_t *hdr;
    mca_btl_openib_footer_t *ftr;
    struct ibv_recv_wr rd_desc;
    uint8_t qp_idx;
} mca_btl_openib_recv_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_t);

#define to_recv_frag(f) ((mca_btl_openib_recv_frag_t*)(f))

typedef struct mca_btl_openib_out_frag_t mca_btl_openib_put_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_put_frag_t);

#define to_put_frag(f) ((mca_btl_openib_put_frag_t*)(f))

typedef struct mca_btl_openib_get_frag_t {
    mca_btl_openib_in_frag_t super;
    struct ibv_send_wr sr_desc;
} mca_btl_openib_get_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_get_frag_t);

#define to_get_frag(f) ((mca_btl_openib_get_frag_t*)(f))

typedef struct mca_btl_openib_send_frag_t mca_btl_openib_send_control_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_send_control_frag_t);

#define to_send_control_frag(f) ((mca_btl_openib_send_control_frag_t*)(f))

typedef struct mca_btl_openib_coalesced_frag_t {
    mca_btl_openib_frag_t super;
    mca_btl_openib_send_frag_t *send_frag;
    mca_btl_openib_header_coalesced_t *hdr;
} mca_btl_openib_coalesced_frag_t;
OBJ_CLASS_DECLARATION(mca_btl_openib_coalesced_frag_t);

#define to_coalesced_frag(f) ((mca_btl_openib_coalesced_frag_t*)(f))

/*
 * Allocate an IB send descriptor
 *
 */

static inline mca_btl_openib_send_control_frag_t *
alloc_control_frag(mca_btl_openib_module_t *btl)
{
    int rc;
    ompi_free_list_item_t *item;

    OMPI_FREE_LIST_WAIT(&btl->device->send_free_control, item, rc);

    return to_send_control_frag(item);
}

static inline uint8_t frag_size_to_order(mca_btl_openib_module_t* btl,
        size_t size)
{
    int qp;
    for(qp = 0; qp < mca_btl_openib_component.num_qps; qp++)
        if(mca_btl_openib_component.qp_infos[qp].size >= size)
            return qp;

    return MCA_BTL_NO_ORDER;
}

static inline mca_btl_openib_com_frag_t *alloc_send_user_frag(void)
{
    int rc;
    ompi_free_list_item_t *item;

    OMPI_FREE_LIST_GET(&mca_btl_openib_component.send_user_free, item, rc);

    return to_com_frag(item);
}

static inline mca_btl_openib_com_frag_t *alloc_recv_user_frag(void)
{
    int rc;
    ompi_free_list_item_t *item;

    OMPI_FREE_LIST_GET(&mca_btl_openib_component.recv_user_free, item, rc);

    return to_com_frag(item);
}

static inline mca_btl_openib_coalesced_frag_t *alloc_coalesced_frag(void)
{
    int rc;
    ompi_free_list_item_t *item;

    OMPI_FREE_LIST_GET(&mca_btl_openib_component.send_free_coalesced, item, rc);

    return to_coalesced_frag(item);
}

#define MCA_BTL_IB_FRAG_RETURN(frag)                                    \
    do {                                                                \
        OMPI_FREE_LIST_RETURN(to_base_frag(frag)->list,                 \
                (ompi_free_list_item_t*)(frag));                        \
    } while(0);

#define MCA_BTL_OPENIB_CLEAN_PENDING_FRAGS(list)                         \
    while(!opal_list_is_empty(list)){                                    \
        opal_list_item_t *frag_item;                                     \
        frag_item = opal_list_remove_first(list);                        \
        MCA_BTL_IB_FRAG_RETURN(frag_item);                               \
    }                                                                    \

struct mca_btl_openib_module_t;

struct mca_btl_openib_frag_init_data_t {
    uint8_t order;
    ompi_free_list_t* list;
};
typedef struct mca_btl_openib_frag_init_data_t mca_btl_openib_frag_init_data_t;

void mca_btl_openib_frag_init(ompi_free_list_item_t* item, void* ctx);


END_C_DECLS
#endif