1. Jeff Squyres
  2. mpi3-fortran

Source

mpi3-fortran / ompi / mca / btl / udapl / btl_udapl.h

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2009 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
 * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
 *
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */
/**
 * @file
 */
#ifndef MCA_BTL_UDAPL_H
#define MCA_BTL_UDAPL_H

#include "ompi_config.h"
#include <sys/types.h>
#include <string.h>
#include <dat/udat.h>

/* Open MPI includes */
#include "opal/class/opal_pointer_array.h"
#include "opal/mca/event/event.h"
#include "ompi/class/ompi_free_list.h"
#include "ompi/mca/btl/btl.h"
#include "ompi/mca/btl/base/base.h"
#include "ompi/mca/btl/base/btl_base_error.h"
#include "ompi/mca/mpool/mpool.h" 
#include "ompi/mca/btl/btl.h"
#include "btl_udapl_endpoint.h"

BEGIN_C_DECLS


/**
 * uDAPL BTL component.
 */

struct mca_btl_udapl_component_t {
    mca_btl_base_component_2_0_0_t super;  /**< base BTL component */ 
    
    int32_t udapl_verbosity; /**< report out level, see
                                "Report Out from uDAPL BTL" below for details. */
    size_t  udapl_num_btls; /**< number of hcas available to the uDAPL component */
    size_t  udapl_max_btls; /**< maximum number of supported hcas */
    struct  mca_btl_udapl_module_t **udapl_btls; /**< array of available BTL modules */
    int32_t udapl_num_recvs;    /**< number of recv buffers to keep posted */
    int32_t udapl_num_sends;    /**< number of sends to post on endpoint */
    int32_t udapl_sr_win;       /**< number of fragments recieved before
                                   returning credits to sender */
    uint32_t udapl_timeout;      /**< connection timeout, in microseconds */
    size_t udapl_eager_frag_size;
    size_t udapl_max_frag_size;
    size_t udapl_eager_rdma_frag_size; /* size of the rdma fragement including data
                                        * payload space
                                        */
    
    int udapl_free_list_num;   /**< initial size of free lists */
    int udapl_free_list_max;   /**< maximum size of free lists */
    int udapl_free_list_inc;   /**< number of elements to alloc when growing */
    int32_t udapl_use_eager_rdma; /**< turn rdma for small msgs on/off */
    int32_t udapl_eager_rdma_num;  /**< number of rdma buffers allocated
                                      for short messages */
    int32_t udapl_max_eager_rdma_peers; /**< maximum number of peers allowed to
                                           use RDMA for short messages (cap)
                                        */
    int32_t udapl_eager_rdma_win; /**< number of eager RDMA fragments
                                    recieved before returning credits to
                                    sender */
    int32_t udapl_conn_priv_data; /**< use connect priv data for proc data */
    int32_t udapl_async_events;  /**< dequeue asynchronous events */
    int32_t udapl_buffer_alignment;  /**< preferred communication buffer alignment, in bytes */
    opal_list_t udapl_procs;   /**< list of udapl proc structures */
    opal_mutex_t udapl_lock;   /**< lock for accessing module state */
    char* udapl_mpool_name;    /**< name of memory pool */ 
    int32_t udapl_compare_subnet;/**< whether to compare with netmask or not */
    char *if_include;
    char **if_include_list;
    char *if_exclude;
    char **if_exclude_list;
    char **if_list;            /* used for checking entries not found */
    int32_t ro_aware_system;   /* default 0; 1 if relaxed ordered platform */
}; 
typedef struct mca_btl_udapl_component_t mca_btl_udapl_component_t;

OMPI_MODULE_DECLSPEC extern mca_btl_udapl_component_t mca_btl_udapl_component;



/**
 * BTL Module Interface
 */
struct mca_btl_udapl_module_t {
    mca_btl_base_module_t  super;  /**< base BTL interface */
    mca_btl_udapl_addr_t udapl_addr;

    /* uDAPL interface and other handles */
    DAT_IA_HANDLE udapl_ia;
    DAT_PZ_HANDLE udapl_pz;
    DAT_PSP_HANDLE udapl_psp;
    DAT_IA_ATTR udapl_ia_attr;
    
    /* event dispatchers - async, data transfer, connection negotiation */
    DAT_EVD_HANDLE udapl_evd_async;
    DAT_EVD_HANDLE udapl_evd_dto;
    DAT_EVD_HANDLE udapl_evd_conn;
    DAT_EP_PARAM   udapl_ep_param;

    /* free list of fragment descriptors */
    ompi_free_list_t udapl_frag_eager;
    ompi_free_list_t udapl_frag_eager_recv;
    ompi_free_list_t udapl_frag_max;
    ompi_free_list_t udapl_frag_max_recv;
    ompi_free_list_t udapl_frag_user;
    ompi_free_list_t udapl_frag_control;
    
    opal_mutex_t udapl_lock;    /* lock for accessing module state */
    opal_mutex_t udapl_eager_rdma_lock;         /* eager rdma lock  */
    int32_t udapl_eager_rdma_endpoint_count;   /* count of the number of
                                                 * endpoints in
                                                 * udapl_eager_rdma_endpoints
                                                 */
    opal_pointer_array_t *udapl_eager_rdma_endpoints;   /* array of endpoints
                                                         * with eager rdma
                                                         * connections
                                                         */
    int32_t udapl_async_events;
    int32_t udapl_connect_inprogress;
    int32_t udapl_num_peers;

    /* module specific limits */
    int     udapl_async_evd_qlen;
    int     udapl_conn_evd_qlen;
    int     udapl_dto_evd_qlen;
    int udapl_max_request_dtos; /**< maximum number of outstanding consumer
                                       submitted sends and rdma operations, see
                                       section 6.6.6 of uDAPL Spec */
    int udapl_max_recv_dtos;    /**< maximum number of outstanding consumer
                                       submitted recv operations, see section
                                       6.6.6 of uDAPL Spec */
    uint32_t udapl_if_mask;     /**< netmask value btl module */
}; 
typedef struct mca_btl_udapl_module_t mca_btl_udapl_module_t;
extern mca_btl_udapl_module_t mca_btl_udapl_module;

struct mca_btl_udapl_reg_t {
    mca_mpool_base_registration_t base;
    DAT_LMR_HANDLE lmr;          /* local memory region (LMR) */
    DAT_LMR_TRIPLET lmr_triplet; /* LMR triplet - context, address, length */
    DAT_RMR_CONTEXT rmr_context; /* remote memory region context handle */

};
typedef struct mca_btl_udapl_reg_t mca_btl_udapl_reg_t;

/**
  * Report Out from uDAPL BTL
  *
  * - BTL_ERROR() : Use to report out errors from uDAPL BTL. These are
  * critical errors which will most likely cause the program to fail so
  * this message should always be reported to the user. Defined in
  * btl/base/btl_base_error.h.
  *   Example:
  *       dat_strerror(rc, (const char**)&major, (const char**)&minor);
  *       BTL_ERROR(("ERROR: %s %s %s\n", "dat_cr_accept", major, minor));
  * 
  * - BTL_UDAPL_VERBOSE_OUTPUT() : Use to output different levels
  *   of verbosity to the user. See Note below.
  *   Example:
  *       BTL_UDAPL_VERBOSE_OUTPUT(VERBOSE_DIAGNOSE,
  *           ("WARNING: don't %s", "jump"));
  *
  * - BTL_UDAPL_VERBOSE_HELP() : Use output information as defined in
  *   uDAPL BTL help file (help-mpi-btl-udapl.txt). See Note below.
  *   Example:
  *       BTL_UDAPL_VERBOSE_HELP(VERBOSE_SHOW_HELP,
  *           ("help-mpi-btl-udapl.txt",
  *           "invalid num rdma segments", true, 22));
  *
  * Note : - Verbose levels are defined below. These levels are
  *        controlled by the mca parameter "btl_udapl_verbose".
  *        The verbose level is set to 10 by default so that critical
  *        error and useful help information will appear. Which ever value
  *        this param is set to, those messages as well as any lower level
  *        verbose messages will be reported.
  *        - Setting "btl_udapl_verbose" to "-1" will turn off all
  *        messages reported by the use of BTL_UDAPL_VERBOSE_*().
  *        - These macros should not be used in a critical path as they
  *        are always included in the compiled code.
  *        - These macros rely on the use of paranthesis around the "args"
  *        value.
  *
  * Values used with BTL_UDAPL_VERBOSE_*():
  *
  *  -   0: critical user information; should always be reported;
  *         on by default
  *  -  10: useful help messages that would be reported from
  *         "help-mpi-btl-udapl.txt"; accessed from
  *         BTL_UDAPL_VERBOSE_HELP(); on by default
  *  -  20: general execution diagnostic information;
  *         may be useful to user or btl developer
  *  -  30: basic debugging/diagnostic information
  *  -  90: useful only to developers
  *  - 100: other components do not appear to go beyond 100 for verbose
  *         levels so noting here as the max for future reference
  */
#define VERBOSE_CRITICAL 0    
#define VERBOSE_SHOW_HELP 10
#define VERBOSE_INFORM 20
#define VERBOSE_DIAGNOSE 30
#define VERBOSE_DEVELOPER 90

#define BTL_UDAPL_VERBOSE_OUTPUT(verbose_level, args)               \
do {                                                                \
    if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
        BTL_OUTPUT(args);                                           \
    }                                                               \
} while(0);

#define BTL_UDAPL_VERBOSE_HELP(verbose_level, args)                 \
do {                                                                \
    if (verbose_level <= mca_btl_udapl_component.udapl_verbosity) { \
        ompi_show_help args;                                        \
    }                                                               \
} while(0);

    
/*
 * Report a uDAPL error - for debugging
 */

#if OPAL_ENABLE_DEBUG
extern void mca_btl_udapl_error(DAT_RETURN ret, char* str);

#define MCA_BTL_UDAPL_ERROR(ret, str) \
    mca_btl_udapl_error((ret), (str));

#else
#define MCA_BTL_UDAPL_ERROR(ret, str)
#endif


/**
 * Register uDAPL component parameters with the MCA framework
 */
extern int mca_btl_udapl_component_open(void);

/**
 * Any final cleanup before being unloaded.
 */
extern int mca_btl_udapl_component_close(void);

/**
 * uDAPL component initialization.
 * 
 * @param num_btl_modules (OUT)           Number of BTLs returned in BTL array.
 * @param allow_multi_user_threads (OUT)  Flag indicating wether BTL supports user threads (TRUE)
 * @param have_hidden_threads (OUT)       Flag indicating wether BTL uses threads (TRUE)
 */
extern mca_btl_base_module_t** mca_btl_udapl_component_init(
    int *num_btl_modules, 
    bool allow_multi_user_threads,
    bool have_hidden_threads
);


/**
 * uDAPL component progress.
 */

extern int mca_btl_udapl_component_progress(void);


/**
 * Initialize resources for a new BTL/uDAPL IA
 *
 * @param ia_name   Name of uDAPL interface adapter
 * @param btl       BTL instance.
 * @return          OMPI_SUCCESS or error status on failure.
 */

extern int mca_btl_udapl_init(
    DAT_NAME_PTR ia_name,
    struct mca_btl_udapl_module_t* btl
);


/**
 * Cleanup any resources held by the BTL.
 * 
 * @param btl  BTL instance.
 * @return     OMPI_SUCCESS or error status on failure.
 */

extern int mca_btl_udapl_finalize(
    struct mca_btl_base_module_t* btl
);


/**
 * PML->BTL notification of change in the process list.
 * 
 * @param btl (IN)
 * @param nprocs (IN)     Number of processes
 * @param procs (IN)      Set of processes
 * @param peers (OUT)     Set of (optional) peer addressing info.
 * @param peers (IN/OUT)  Set of processes that are reachable via this BTL.
 * @return     OMPI_SUCCESS or error status on failure.
 * 
 */

extern int mca_btl_udapl_add_procs(
    struct mca_btl_base_module_t* btl,
    size_t nprocs,
    struct ompi_proc_t **procs,
    struct mca_btl_base_endpoint_t** peers,
    opal_bitmap_t* reachable
);

/**
 * PML->BTL notification of change in the process list.
 *
 * @param btl (IN)     BTL instance
 * @param nproc (IN)   Number of processes.
 * @param procs (IN)   Set of processes.
 * @param peers (IN)   Set of peer data structures.
 * @return             Status indicating if cleanup was successful
 *
 */

extern int mca_btl_udapl_del_procs(
    struct mca_btl_base_module_t* btl,
    size_t nprocs,
    struct ompi_proc_t **procs,
    struct mca_btl_base_endpoint_t** peers
);


/**
 * Initiate an asynchronous send.
 *
 * @param btl (IN)         BTL module
 * @param endpoint (IN)    BTL addressing information
 * @param descriptor (IN)  Description of the data to be transferred
 * @param tag (IN)         The tag value used to notify the peer.
 */

extern int mca_btl_udapl_send(
    struct mca_btl_base_module_t* btl,
    struct mca_btl_base_endpoint_t* btl_peer,
    struct mca_btl_base_descriptor_t* descriptor, 
    mca_btl_base_tag_t tag
);


/**
 * Initiate an asynchronous put.
 *
 * @param btl (IN)         BTL module
 * @param endpoint (IN)    BTL addressing information
 * @param descriptor (IN)  Description of the data to be transferred
 */
                                                                                                    
extern int mca_btl_udapl_put(
    struct mca_btl_base_module_t* btl,
    struct mca_btl_base_endpoint_t* btl_peer,
    struct mca_btl_base_descriptor_t* decriptor
);


/**
 * Initiate an asynchronous get.
 *
 * @param btl (IN)         BTL module
 * @param endpoint (IN)    BTL addressing information
 * @param descriptor (IN)  Description of the data to be transferred
 */
                                                                                                    
extern int mca_btl_udapl_get(
    struct mca_btl_base_module_t* btl,
    struct mca_btl_base_endpoint_t* btl_peer,
    struct mca_btl_base_descriptor_t* decriptor
);

/**
 * Allocate a descriptor with a segment of the requested size.
 * Note that the BTL layer may choose to return a smaller size
 * if it cannot support the request.
 *
 * @param btl (IN)      BTL module
 * @param size (IN)     Request segment size.
 */

extern mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
    struct mca_btl_base_module_t* btl,
    struct mca_btl_base_endpoint_t* endpoint,
    uint8_t order,
    size_t size,
    uint32_t flags); 


/**
 * Return a segment allocated by this BTL.
 *
 * @param btl (IN)      BTL module
 * @param descriptor (IN)  Allocated descriptor.
 */

extern int mca_btl_udapl_free(
    struct mca_btl_base_module_t* btl, 
    mca_btl_base_descriptor_t* des); 
    

/**
 * Prepare a descriptor for send/rdma using the supplied
 * convertor. If the convertor references data that is contigous,
 * the descriptor may simply point to the user buffer. Otherwise,
 * this routine is responsible for allocating buffer space and
 * packing if required.
 *
 * @param btl (IN)          BTL module
 * @param endpoint (IN)     BTL peer addressing
 * @param convertor (IN)    Data type convertor
 * @param reserve (IN)      Additional bytes requested by upper layer to precede user data
 * @param size (IN/OUT)     Number of bytes to prepare (IN), number of bytes actually prepared (OUT) 
*/

mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
    struct mca_btl_base_module_t* btl,
    struct mca_btl_base_endpoint_t* peer,
    struct mca_mpool_base_registration_t*,
    struct opal_convertor_t* convertor,
    uint8_t order,
    size_t reserve,
    size_t* size,
    uint32_t flags
);

extern mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst( 
    struct mca_btl_base_module_t* btl, 
    struct mca_btl_base_endpoint_t* peer,
    struct mca_mpool_base_registration_t*,
    struct opal_convertor_t* convertor,
    uint8_t order,
    size_t reserve,
    size_t* size,
    uint32_t flags); 

    /**
     * Fault Tolerance Event Notification Function
     * @param state Checkpoint Stae
     * @return OMPI_SUCCESS or failure status
     */
int mca_btl_udapl_ft_event(int state);

END_C_DECLS
#endif