Source

mpi3-fortran / ompi / mca / coll / sm / coll_sm.h

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
/* 
 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2006 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */
/** @file */

#ifndef MCA_COLL_SM_EXPORT_H
#define MCA_COLL_SM_EXPORT_H

#include "ompi_config.h"

#include "mpi.h"
#include "opal/mca/mca.h"
#include "opal/datatype/opal_convertor.h"
#include "ompi/mca/coll/coll.h"
#include "ompi/mca/common/sm/common_sm.h"

BEGIN_C_DECLS

/* Attempt to give some sort of progress / fairness if we're blocked
   in an sm collective for a long time: call opal_progress once in a
   great while.  Use a "goto" label for expdiency to exit loops. */
#define SPIN_CONDITION_MAX 100000
#define SPIN_CONDITION(cond, exit_label) \
  do { int i; \
       if (cond) goto exit_label; \
       for (i = 0; i < SPIN_CONDITION_MAX; ++i) { \
           if (cond) { goto exit_label; } \
       } \
       opal_progress(); \
  } while (1); \
  exit_label: 

    /** 
     * Structure to hold the sm coll component.  First it holds the
     * base coll component, and then holds a bunch of
     * sm-coll-component-specific stuff (e.g., current MCA param
     * values). 
     */
    typedef struct mca_coll_sm_component_t {
        /** Base coll component */
        mca_coll_base_component_2_0_0_t super;

        /** MCA parameter: Priority of this component */
        int sm_priority;

        /** MCA parameter: Length of a cache line or page (in bytes) */
        int sm_control_size;

        /** MCA parameter: Number of "in use" flags in each
            communicator's area in the data mpool */
        int sm_comm_num_in_use_flags;

        /** MCA parameter: Number of segments for each communicator in
            the data mpool */
        int sm_comm_num_segments;

        /** MCA parameter: Fragment size for data */
        int sm_fragment_size;

        /** MCA parameter: Degree of tree for tree-based collectives */
        int sm_tree_degree;

        /** MCA parameter: Number of processes to use in the
            calculation of the "info" MCA parameter */
        int sm_info_comm_size;

        /******* end of MCA params ********/

        /** How many fragment segments are protected by a single
            in-use flags.  This is solely so that we can only perform
            the division once and then just use the value without
            having to re-calculate. */
        int sm_segs_per_inuse_flag;
    } mca_coll_sm_component_t;

    /**
     * Structure for representing a node in the tree
     */
    typedef struct mca_coll_sm_tree_node_t {
        /** Arbitrary ID number, starting from 0 */
        int mcstn_id;
        /** Pointer to parent, or NULL if root */
        struct mca_coll_sm_tree_node_t *mcstn_parent;
        /** Number of children, or 0 if a leaf */
        int mcstn_num_children;
        /** Pointer to an array of children, or NULL if 0 ==
            mcstn_num_children */
        struct mca_coll_sm_tree_node_t **mcstn_children;
    } mca_coll_sm_tree_node_t;

    /**
     * Simple structure comprising the "in use" flags.  Contains two
     * members: the number of processes that are currently using this
     * set of segments and the operation number of the current
     * operation.
     */
    typedef struct mca_coll_sm_in_use_flag_t {
        /** Number of processes currently using this set of
            segments */
        volatile uint32_t mcsiuf_num_procs_using;
        /** Must match data->mcb_count */
        volatile uint32_t mcsiuf_operation_count;
    } mca_coll_sm_in_use_flag_t;

    /**
     * Structure containing pointers to various arrays of data in the
     * per-communicator shmem data segment (one of these indexes a
     * single segment in the per-communicator shmem data segment).
     * Nothing is hard-coded because all the array lengths and
     * displacements of the pointers all depend on how many processes
     * are in the communicator.
     */
    typedef struct mca_coll_sm_data_index_t {
        /** Pointer to beginning of control data */
        uint32_t volatile *mcbmi_control;
        /** Pointer to beginning of message fragment data */
        char *mcbmi_data;
    } mca_coll_sm_data_index_t;

    /**
     * Structure for the sm coll module to hang off the communicator.
     * Contains communicator-specific information, including pointers
     * into the per-communicator shmem data data segment for this
     * comm's sm collective operations area.
     */
    typedef struct mca_coll_sm_comm_t {
        /* Meta data that we get back from the common mmap allocation
           function */
        mca_common_sm_module_t *sm_bootstrap_meta;

        /** Pointer to my barrier control pages (odd index pages are
            "in", even index pages are "out") */
        uint32_t *mcb_barrier_control_me;

        /** Pointer to my parent's barrier control pages (will be NULL
            for communicator rank 0; odd index pages are "in", even
            index pages are "out") */
        uint32_t *mcb_barrier_control_parent;

        /** Pointers to my childrens' barrier control pages (they're
            contiguous in memory, so we only point to the base -- the
            number of children is in my entry in the mcb_tree); will
            be NULL if this process has no children (odd index pages
            are "in", even index pages are "out") */
        uint32_t *mcb_barrier_control_children;

        /** Number of barriers that we have executed (i.e., which set
            of barrier buffers to use). */
        int mcb_barrier_count;

        /** "In use" flags indicating which segments are available */
        mca_coll_sm_in_use_flag_t *mcb_in_use_flags;

        /** Array of indexes into the per-communicator shmem data
            segment for control and data fragment passing (containing
            pointers to each segments control and data areas). */
        mca_coll_sm_data_index_t *mcb_data_index;

        /** Array of graph nodes representing the tree used for
            communications */
        mca_coll_sm_tree_node_t *mcb_tree;

        /** Operation number (i.e., which segment number to use) */
        uint32_t mcb_operation_count;
    } mca_coll_sm_comm_t;

    /** Coll sm module */
    typedef struct mca_coll_sm_module_t {
        /** Base module */
	mca_coll_base_module_t super;

        /* Whether this module has been lazily initialized or not yet */
        bool enabled;

        /* Data that hangs off the communicator */
	mca_coll_sm_comm_t *sm_comm_data;

        /* Underlying reduce function and module */
	mca_coll_base_module_reduce_fn_t previous_reduce;
	mca_coll_base_module_t *previous_reduce_module;
    } mca_coll_sm_module_t;
    OBJ_CLASS_DECLARATION(mca_coll_sm_module_t);
    
    /**
     * Global component instance
     */
    OMPI_MODULE_DECLSPEC extern mca_coll_sm_component_t mca_coll_sm_component;

    /*
     * coll module functions
     */
    int mca_coll_sm_init_query(bool enable_progress_threads,
			       bool enable_mpi_threads);

    mca_coll_base_module_t *
    mca_coll_sm_comm_query(struct ompi_communicator_t *comm, int *priority);
    
    /* Lazily enable a module (since it involves expensive/slow mmap
       allocation, etc.) */
    int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
                                 struct ompi_communicator_t *comm);

    int mca_coll_sm_allgather_intra(void *sbuf, int scount, 
				    struct ompi_datatype_t *sdtype, 
				    void *rbuf, int rcount, 
				    struct ompi_datatype_t *rdtype, 
				    struct ompi_communicator_t *comm,
				    mca_coll_base_module_t *module);

    int mca_coll_sm_allgatherv_intra(void *sbuf, int scount, 
				     struct ompi_datatype_t *sdtype, 
				     void * rbuf, int *rcounts, int *disps, 
				     struct ompi_datatype_t *rdtype, 
				     struct ompi_communicator_t *comm,
				     mca_coll_base_module_t *module);
    int mca_coll_sm_allreduce_intra(void *sbuf, void *rbuf, int count, 
				    struct ompi_datatype_t *dtype, 
				    struct ompi_op_t *op, 
				    struct ompi_communicator_t *comm,
				    mca_coll_base_module_t *module);
    int mca_coll_sm_alltoall_intra(void *sbuf, int scount, 
				   struct ompi_datatype_t *sdtype, 
				   void* rbuf, int rcount, 
				   struct ompi_datatype_t *rdtype, 
				   struct ompi_communicator_t *comm,
				   mca_coll_base_module_t *module);
    int mca_coll_sm_alltoallv_intra(void *sbuf, int *scounts, int *sdisps, 
				    struct ompi_datatype_t *sdtype, 
				    void *rbuf, int *rcounts, int *rdisps, 
				    struct ompi_datatype_t *rdtype, 
				    struct ompi_communicator_t *comm,
				    mca_coll_base_module_t *module);
    int mca_coll_sm_alltoallw_intra(void *sbuf, int *scounts, int *sdisps, 
				    struct ompi_datatype_t **sdtypes, 
				    void *rbuf, int *rcounts, int *rdisps, 
				    struct ompi_datatype_t **rdtypes, 
				    struct ompi_communicator_t *comm,
				    mca_coll_base_module_t *module);
    int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
				  mca_coll_base_module_t *module);
    int mca_coll_sm_bcast_intra(void *buff, int count, 
				struct ompi_datatype_t *datatype,
				int root, 
				struct ompi_communicator_t *comm,
				mca_coll_base_module_t *module);
    int mca_coll_sm_bcast_log_intra(void *buff, int count, 
				    struct ompi_datatype_t *datatype, 
				    int root, 
				    struct ompi_communicator_t *comm, 
				    mca_coll_base_module_t *module);
    int mca_coll_sm_exscan_intra(void *sbuf, void *rbuf, int count, 
				 struct ompi_datatype_t *dtype, 
				 struct ompi_op_t *op, 
				 struct ompi_communicator_t *comm, 
				 mca_coll_base_module_t *module);
    int mca_coll_sm_gather_intra(void *sbuf, int scount, 
				 struct ompi_datatype_t *sdtype, void *rbuf, 
				 int rcount, struct ompi_datatype_t *rdtype, 
				 int root, struct ompi_communicator_t *comm,
				 mca_coll_base_module_t *module);
    int mca_coll_sm_gatherv_intra(void *sbuf, int scount, 
				  struct ompi_datatype_t *sdtype, void *rbuf, 
				  int *rcounts, int *disps, 
				  struct ompi_datatype_t *rdtype, int root, 
				  struct ompi_communicator_t *comm,
				  mca_coll_base_module_t *module);
    int mca_coll_sm_reduce_intra(void *sbuf, void* rbuf, int count, 
				 struct ompi_datatype_t *dtype, 
				 struct ompi_op_t *op, 
				 int root,
				 struct ompi_communicator_t *comm,
				 mca_coll_base_module_t *module);
    int mca_coll_sm_reduce_log_intra(void *sbuf, void* rbuf, int count, 
				     struct ompi_datatype_t *dtype, 
				     struct ompi_op_t *op, 
				     int root, 
				     struct ompi_communicator_t *comm,
				     mca_coll_base_module_t *module);
    int mca_coll_sm_reduce_scatter_intra(void *sbuf, void *rbuf, 
					 int *rcounts, 
					 struct ompi_datatype_t *dtype, 
					 struct ompi_op_t *op, 
					 struct ompi_communicator_t *comm,
					 mca_coll_base_module_t *module);
    int mca_coll_sm_scan_intra(void *sbuf, void *rbuf, int count, 
			       struct ompi_datatype_t *dtype, 
			       struct ompi_op_t *op, 
			       struct ompi_communicator_t *comm,
			       mca_coll_base_module_t *module);
    int mca_coll_sm_scatter_intra(void *sbuf, int scount, 
				  struct ompi_datatype_t *sdtype, void *rbuf, 
				  int rcount, struct ompi_datatype_t *rdtype, 
				  int root, struct ompi_communicator_t *comm,
				  mca_coll_base_module_t *module);
    int mca_coll_sm_scatterv_intra(void *sbuf, int *scounts, int *disps, 
				   struct ompi_datatype_t *sdtype, 
				   void* rbuf, int rcount, 
				   struct ompi_datatype_t *rdtype, int root, 
				   struct ompi_communicator_t *comm,
				   mca_coll_base_module_t *module);

    int mca_coll_sm_ft_event(int state);
    
/**
 * Global variables used in the macros (essentially constants, so
 * these are thread safe)
 */
extern uint32_t mca_coll_sm_one;


/**
 * Macro to setup flag usage
 */
#define FLAG_SETUP(flag_num, flag, data) \
    (flag) = (mca_coll_sm_in_use_flag_t*) \
        (((char *) (data)->mcb_in_use_flags) + \
        ((flag_num) * mca_coll_sm_component.sm_control_size))

/**
 * Macro to wait for the in-use flag to become idle (used by the root)
 */
#define FLAG_WAIT_FOR_IDLE(flag, label) \
    SPIN_CONDITION(0 == (flag)->mcsiuf_num_procs_using, label)

/**
 * Macro to wait for a flag to indicate that it's ready for this
 * operation (used by non-root processes to know when FLAG_SET() has
 * been called)
 */
#define FLAG_WAIT_FOR_OP(flag, op, label) \
    SPIN_CONDITION((op) == flag->mcsiuf_operation_count, label)

/**
 * Macro to set an in-use flag with relevant data to claim it
 */
#define FLAG_RETAIN(flag, num_procs, op_count) \
    (flag)->mcsiuf_num_procs_using = (num_procs); \
    (flag)->mcsiuf_operation_count = (op_count)

/**
 * Macro to release an in-use flag from this process
 */
#define FLAG_RELEASE(flag) \
    opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1)

/**
 * Macro to copy a single segment in from a user buffer to a shared
 * segment
 */
#define COPY_FRAGMENT_IN(convertor, index, rank, iov, max_data) \
    (iov).iov_base = \
        (index)->mcbmi_data + \
        ((rank) * mca_coll_sm_component.sm_fragment_size); \
    (iov).iov_len = (max_data); \
    opal_convertor_pack(&(convertor), &(iov), &mca_coll_sm_one, \
                        &(max_data) )

/**
 * Macro to copy a single segment out from a shared segment to a user
 * buffer
 */
#define COPY_FRAGMENT_OUT(convertor, src_rank, index, iov, max_data) \
    (iov).iov_base = (((char*) (index)->mcbmi_data) + \
                       ((src_rank) * (mca_coll_sm_component.sm_fragment_size))); \
    (iov).iov_len = (max_data); \
    opal_convertor_unpack(&(convertor), &(iov), &mca_coll_sm_one, \
                          &(max_data) )

/**
 * Macro to memcpy a fragment between one shared segment and another
 */
#define COPY_FRAGMENT_BETWEEN(src_rank, dest_rank, index, len) \
    memcpy(((index)->mcbmi_data + \
            ((dest_rank) * mca_coll_sm_component.sm_fragment_size)), \
           ((index)->mcbmi_data + \
            ((src_rank) * \
             mca_coll_sm_component.sm_fragment_size)), \
           (len))

/** 
 * Macro to tell children that a segment is ready (normalize
 * the child's ID based on the shift used to calculate the "me" node
 * in the tree).  Used in fan out opertations.
 */
#define PARENT_NOTIFY_CHILDREN(children, num_children, index, value) \
    do { \
        for (i = 0; i < (num_children); ++i) { \
            *((size_t*) \
              (((char*) index->mcbmi_control) + \
               (mca_coll_sm_component.sm_control_size * \
                (((children)[i]->mcstn_id + root) % size)))) = (value); \
        } \
    } while (0)

/**
 * Macro for childen to wait for parent notification (use real rank).
 * Save the value passed and then reset it when done.  Used in fan out
 * operations.
 */
#define CHILD_WAIT_FOR_NOTIFY(rank, index, value, label) \
    do { \
        uint32_t volatile *ptr = ((uint32_t*) \
                                  (((char*) index->mcbmi_control) + \
                                   ((rank) * mca_coll_sm_component.sm_control_size))); \
        SPIN_CONDITION(0 != *ptr, label); \
        (value) = *ptr; \
        *ptr = 0; \
    } while (0)

/**
 * Macro for children to tell parent that the data is ready in their
 * segment.  Used for fan in operations.
 */
#define CHILD_NOTIFY_PARENT(child_rank, parent_rank, index, value) \
    ((size_t volatile *) \
     (((char*) (index)->mcbmi_control) + \
      (mca_coll_sm_component.sm_control_size * \
       (parent_rank))))[(child_rank)] = (value)

/**
 * Macro for parent to wait for a specific child to tell it that the
 * data is in the child's segment.  Save the value when done.  Used
 * for fan in operations.
 */
#define PARENT_WAIT_FOR_NOTIFY_SPECIFIC(child_rank, parent_rank, index, value, label) \
    do { \
        size_t volatile *ptr = ((size_t volatile *) \
                                (((char*) index->mcbmi_control) + \
                                 (mca_coll_sm_component.sm_control_size * \
                                  (parent_rank)))) + child_rank; \
        SPIN_CONDITION(0 != *ptr, label); \
        (value) = *ptr; \
        *ptr = 0; \
    } while (0)

END_C_DECLS

#endif /* MCA_COLL_SM_EXPORT_H */