Source

mpi3-fortran / opal / include / opal / sys / atomic.h

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
/*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2006 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
 * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
 * 
 * $HEADER$
 */

/** @file
 *
 * Atomic operations.
 *
 * This API is patterned after the FreeBSD kernel atomic interface
 * (which is influenced by Intel's ia64 architecture).  The
 * FreeBSD interface is documented at
 *
 * http://www.freebsd.org/cgi/man.cgi?query=atomic&sektion=9
 *
 * Only the necessary subset of functions are implemented here.
 *
 * The following #defines will be true / false based on
 * assembly support:
 *
 *  - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
 *  - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
 *  - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/cmpset can be done "atomicly"
 *  - \c OPAL_HAVE_ATOMIC_MATH_64 if 32 bit add/sub/cmpset can be done "atomicly"
 *
 * Note that for the Atomic math, atomic add/sub may be implemented as
 * C code using opal_atomic_cmpset.  The appearance of atomic
 * operation will be upheld in these cases.
 */

#ifndef OPAL_SYS_ATOMIC_H
#define OPAL_SYS_ATOMIC_H 1

#include "opal_config.h"

#include "opal/sys/architecture.h"

#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif

/* do some quick #define cleanup in cases where we are doing
   testing... */
#ifdef OMPI_DISABLE_INLINE_ASM
#undef OPAL_C_GCC_INLINE_ASSEMBLY
#define OPAL_C_GCC_INLINE_ASSEMBLY 0
#undef OPAL_C_DEC_INLINE_ASSEMBLY
#define OPAL_C_DEC_INLINE_ASSEMBLY 0
#undef OPAL_C_XLC_INLINE_ASSEMBLY
#define OPAL_C_XLC_INLINE_ASSEMBLY 0
#endif

/* define OMPI_{GCC,DEC,XLC}_INLINE_ASSEMBLY based on the
   OMPI_C_{GCC,DEC,XLC}_INLINE_ASSEMBLY defines and whether we
   are in C or C++ */
#if defined(c_plusplus) || defined(__cplusplus)
/* We no longer support inline assembly for C++ as OPAL is a C-only interface */
#define OMPI_GCC_INLINE_ASSEMBLY 0
#define OMPI_DEC_INLINE_ASSEMBLY 0
#define OMPI_XLC_INLINE_ASSEMBLY 0
#else
#define OMPI_GCC_INLINE_ASSEMBLY OPAL_C_GCC_INLINE_ASSEMBLY
#define OMPI_DEC_INLINE_ASSEMBLY OPAL_C_DEC_INLINE_ASSEMBLY
#define OMPI_XLC_INLINE_ASSEMBLY OPAL_C_XLC_INLINE_ASSEMBLY
#endif


BEGIN_C_DECLS
/**********************************************************************
 *
 * Data structures for atomic ops
 *
 *********************************************************************/
/**
 * Volatile lock object (with optional padding).
 *
 * \note The internals of the lock are included here, but should be
 * considered private.  The implementation currently in use may choose
 * to use an int or unsigned char as the lock value - the user is not
 * informed either way.
 */
struct opal_atomic_lock_t {
    union {
        volatile int32_t lock;     /**< The lock address (an integer) */
        volatile unsigned char sparc_lock; /**< The lock address on sparc */
        char padding[sizeof(int)]; /**< Array for optional padding */
    } u;
};
typedef struct opal_atomic_lock_t opal_atomic_lock_t;

/**********************************************************************
 *
 * Set or unset these macros in the architecture-specific atomic.h
 * files if we need to specify them as inline or non-inline 
 * 
 *********************************************************************/
#if !OMPI_GCC_INLINE_ASSEMBLY
#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 0
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 0
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0
#define OPAL_HAVE_INLINE_ATOMIC_SUB_32 0
#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 0
#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 0
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 0
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0
#else
#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
#define OPAL_HAVE_INLINE_ATOMIC_SUB_32 1
#define OPAL_HAVE_INLINE_ATOMIC_ADD_64 1
#define OPAL_HAVE_INLINE_ATOMIC_SUB_64 1
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 1
#endif

/**********************************************************************
 *
 * Load the appropriate architecture files and set some reasonable
 * default values for our support
 *
 *********************************************************************/
#if defined(DOXYGEN)
/* don't include system-level gorp when generating doxygen files */ 
#elif OPAL_ASSEMBLY_ARCH == OMPI_WINDOWS
/* windows first, as they have API-level primitives for this stuff */
#include "opal/sys/win32/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_ALPHA
#include "opal/sys/alpha/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_AMD64
#include "opal/sys/amd64/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_ARM
#include "opal/sys/arm/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_IA32
#include "opal/sys/ia32/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_IA64
#include "opal/sys/ia64/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_MIPS
#include "opal/sys/mips/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_POWERPC32
#include "opal/sys/powerpc/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_POWERPC64
#include "opal/sys/powerpc/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_SPARC
#include "opal/sys/sparc/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_SPARCV9_32
#include "opal/sys/sparcv9/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_SPARCV9_64
#include "opal/sys/sparcv9/atomic.h"
#elif OPAL_ASSEMBLY_ARCH == OMPI_SYNC_BUILTIN
#include "opal/sys/sync_builtin/atomic.h"
#endif

#ifndef DOXYGEN
/* compare and set operations can't really be emulated from software,
   so if these defines aren't already set, they should be set to 0
   now */
#ifndef OPAL_HAVE_ATOMIC_CMPSET_32
#define OPAL_HAVE_ATOMIC_CMPSET_32 0
#endif
#ifndef OPAL_HAVE_ATOMIC_CMPSET_64
#define OPAL_HAVE_ATOMIC_CMPSET_64 0
#endif
#endif /* DOXYGEN */

/**********************************************************************
 *
 * Memory Barriers - defined here if running doxygen or have barriers
 *                   but can't inline
 *
 *********************************************************************/
#if !defined(OPAL_HAVE_ATOMIC_MEM_BARRIER) && !defined(DOXYGEN)
/* no way to emulate in C code */
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 0
#endif

#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER
/**
 * Memory barrier
 *
 * Will use system-specific features to instruct the processor and
 * memory controller that all writes and reads that have been posted
 * before the call to \c opal_atomic_mb() must appear to have
 * completed before the next read or write.
 *
 * \note This can have some expensive side effects, including flushing
 * the pipeline, preventing the cpu from reordering instructions, and
 * generally grinding the memory controller's performance.  Use only
 * if you need *both* read and write barriers.
 */

#if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
static inline 
#endif
void opal_atomic_mb(void);

/**
 * Read memory barrier
 *
 * Use system-specific features to instruct the processor and memory
 * conrtoller that all reads that have been posted before the call to
 * \c opal_atomic_rmb() must appear to have been completed before the
 * next read.  Nothing is said about the ordering of writes when using
 * \c opal_atomic_rmb().
 */

#if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
static inline 
#endif
void opal_atomic_rmb(void);

/**
 * Write memory barrier.
 *
 * Use system-specific features to instruct the processor and memory
 * conrtoller that all writes that have been posted before the call to
 * \c opal_atomic_wmb() must appear to have been completed before the
 * next write.  Nothing is said about the ordering of reads when using
 * \c opal_atomic_wmb().
 */

#if OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER
static inline 
#endif
void opal_atomic_wmb(void);

#endif /* defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MEM_BARRIER */


/**********************************************************************
 *
 * Atomic spinlocks - always inlined, if have atomic cmpset
 *
 *********************************************************************/

#if !defined(OPAL_HAVE_ATOMIC_SPINLOCKS) && !defined(DOXYGEN)
/* 0 is more like "pending" - we'll fix up at the end after all
   the static inline functions are declared */
#define OPAL_HAVE_ATOMIC_SPINLOCKS 0
#endif

#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)

/**
 * Enumeration of lock states
 */
enum {
    OPAL_ATOMIC_UNLOCKED = 0,
    OPAL_ATOMIC_LOCKED = 1
};


/**
 * Initialize a lock to value
 *
 * @param lock         Address of the lock
 * @param value        Initial value to set lock to
 */
#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
static inline 
#endif
void opal_atomic_init(opal_atomic_lock_t* lock, int32_t value);


/**
 * Try to acquire a lock.
 *
 * @param lock          Address of the lock.
 * @return              0 if the lock was acquired, 1 otherwise.
 */
#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
static inline
#endif
int opal_atomic_trylock(opal_atomic_lock_t *lock);


/**
 * Acquire a lock by spinning.
 *
 * @param lock          Address of the lock.
 */
#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
static inline
#endif
void opal_atomic_lock(opal_atomic_lock_t *lock);


/**
 * Release a lock.
 *
 * @param lock          Address of the lock.
 */
#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
static inline
#endif
void opal_atomic_unlock(opal_atomic_lock_t *lock);


#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
#undef OPAL_HAVE_ATOMIC_SPINLOCKS
#define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
#define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS
#endif

#endif /* OPAL_HAVE_ATOMIC_SPINLOCKS */


/**********************************************************************
 *
 * Atomic math operations
 *
 *********************************************************************/
#if !defined(OPAL_HAVE_ATOMIC_CMPSET_32) && !defined(DOXYGEN)
#define OPAL_HAVE_ATOMIC_CMPSET_32 0
#endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_32

#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline 
#endif
int opal_atomic_cmpset_32(volatile int32_t *addr, int32_t oldval,
                          int32_t newval);

#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline 
#endif
int opal_atomic_cmpset_acq_32(volatile int32_t *addr, int32_t oldval,
                              int32_t newval);

#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline 
#endif
int opal_atomic_cmpset_rel_32(volatile int32_t *addr, int32_t oldval,
                              int32_t newval);
#endif


#if !defined(OPAL_HAVE_ATOMIC_CMPSET_64) && !defined(DOXYGEN)
#define OPAL_HAVE_ATOMIC_CMPSET_64 0
#endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_64

#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline 
#endif
int opal_atomic_cmpset_64(volatile int64_t *addr, int64_t oldval,
                          int64_t newval);

#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline 
#endif
int opal_atomic_cmpset_acq_64(volatile int64_t *addr, int64_t oldval,
                              int64_t newval);

#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline 
#endif
int opal_atomic_cmpset_rel_64(volatile int64_t *addr, int64_t oldval,
                              int64_t newval);

#endif

#if !defined(OPAL_HAVE_ATOMIC_MATH_32) && !defined(DOXYGEN)
  /* define to 0 for these tests.  WIll fix up later. */
  #define OPAL_HAVE_ATOMIC_MATH_32 0
#endif

#if defined(DOXYGEN) ||  OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_CMPSET_32

/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
   a static inline version of it (in assembly).  If it's 0 but
   OPAL_HAVE_ATOMIC_CMPSET_32 is 1, then atomic_impl.h (below) will
   define a static inline version of it (in C, using
   atomic_cmpset_32()).  */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || OPAL_HAVE_ATOMIC_CMPSET_32
static inline
#endif
int32_t opal_atomic_add_32(volatile int32_t *addr, int delta);

/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
   a static inline version of it (in assembly).  If it's 0 but
   OPAL_HAVE_ATOMIC_CMPSET_32 is 1, then atomic_impl.h (below) will
   define a static inline version of it (in C, using
   atomic_cmpset_32()).  */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || OPAL_HAVE_ATOMIC_CMPSET_32
static inline
#endif
int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);

#endif /* OPAL_HAVE_ATOMIC_MATH_32 */

#if ! OPAL_HAVE_ATOMIC_MATH_32
/* fix up the value of ompi_have_atomic_math_32 to allow for C versions */
#undef OPAL_HAVE_ATOMIC_MATH_32
#define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_CMPSET_32
#endif

#ifndef OPAL_HAVE_ATOMIC_MATH_64
/* define to 0 for these tests.  WIll fix up later. */
#define OPAL_HAVE_ATOMIC_MATH_64 0
#endif

#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_CMPSET_64

/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
   a static inline version of it (in assembly).  If it's 0 but
   OPAL_HAVE_ATOMIC_CMPSET_64 is 1, then atomic_impl.h (below) will
   define a static inline version of it (in C, using
   atomic_cmpset_64()).  */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || OPAL_HAVE_ATOMIC_CMPSET_64
static inline
#endif
int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta);

/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
   a static inline version of it (in assembly).  If it's 0 but
   OPAL_HAVE_ATOMIC_CMPSET_64 is 1, then atomic_impl.h (below) will
   define a static inline version of it (in C, using
   atomic_cmpset_64()).  */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || OPAL_HAVE_ATOMIC_CMPSET_64
static inline
#endif
int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta);

#endif /* OPAL_HAVE_ATOMIC_MATH_32 */

#if ! OPAL_HAVE_ATOMIC_MATH_64
/* fix up the value of ompi_have_atomic_math_64 to allow for C versions */
#undef OPAL_HAVE_ATOMIC_MATH_64
#define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_CMPSET_64
#endif

/* provide a size_t add/subtract.  When in debug mode, make it an
 * inline function so that we don't have any casts in the
 *  interface and can catch type errors.  When not in debug mode,
 * just make it a macro, so that there's no performance penalty
 */
#if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
static inline size_t
opal_atomic_add_size_t(volatile size_t *addr, int delta)
{
#if SIZEOF_SIZE_T == 4
    return (size_t) opal_atomic_add_32((int32_t*) addr, delta);
#elif SIZEOF_SIZE_T == 8
    return (size_t) opal_atomic_add_64((int64_t*) addr, delta);
#else
#error "Unknown size_t size"
#endif
}
static inline size_t
opal_atomic_sub_size_t(volatile size_t *addr, int delta)
{
#if SIZEOF_SIZE_T == 4
    return (size_t) opal_atomic_sub_32((int32_t*) addr, delta);
#elif SIZEOF_SIZE_T == 8
    return (size_t) opal_atomic_sub_64((int64_t*) addr, delta);
#else
#error "Unknown size_t size"
#endif
}
#else
#if SIZEOF_SIZE_T == 4
#define opal_atomic_add_size_t(addr, delta) ((size_t) opal_atomic_add_32((int32_t*) addr, delta))
#define opal_atomic_sub_size_t(addr, delta) ((size_t) opal_atomic_sub_32((int32_t*) addr, delta))
#elif SIZEOF_SIZE_T ==8
#define opal_atomic_add_size_t(addr, delta) ((size_t) opal_atomic_add_64((int64_t*) addr, delta))
#define opal_atomic_sub_size_t(addr, delta) ((size_t) opal_atomic_sub_64((int64_t*) addr, delta))
#else
#error "Unknown size_t size"
#endif
#endif

#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
/* these are always done with inline functions, so always mark as
   static inline */
static inline int opal_atomic_cmpset_xx(volatile void* addr, int64_t oldval,
                                        int64_t newval, size_t length);
static inline int opal_atomic_cmpset_acq_xx(volatile void* addr, 
                                            int64_t oldval,  int64_t newval, 
                                            size_t length);
static inline int opal_atomic_cmpset_rel_xx(volatile void* addr, 
                                            int64_t oldval, int64_t newval, 
                                            size_t length);

static inline int opal_atomic_cmpset_ptr(volatile void* addr, 
                                         void* oldval, 
                                         void* newval);
static inline int opal_atomic_cmpset_acq_ptr(volatile void* addr, 
                                             void* oldval, 
                                             void* newval);
static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr, 
                                             void* oldval, 
                                             void* newval);

/**
 * Atomic compare and set of pointer with relaxed semantics. This
 * macro detect at compile time the type of the first argument and
 * choose the correct function to be called.  
 *
 * \note This macro should only be used for integer types.
 *
 * @param addr          Address of <TYPE>.
 * @param oldval        Comparison value <TYPE>.
 * @param newval        New value to set if comparision is true <TYPE>.
 *
 * See opal_atomic_cmpset_* for pseudo-code.
 */
#define opal_atomic_cmpset( ADDR, OLDVAL, NEWVAL )                  \
   opal_atomic_cmpset_xx( (volatile void*)(ADDR), (intptr_t)(OLDVAL), \
                          (intptr_t)(NEWVAL), sizeof(*(ADDR)) )

/**
 * Atomic compare and set of pointer with acquire semantics. This
 * macro detect at compile time the type of the first argument 
 * and choose the correct function to be called.
 *
 * \note This macro should only be used for integer types.
 *
 * @param addr          Address of <TYPE>.
 * @param oldval        Comparison value <TYPE>.
 * @param newval        New value to set if comparision is true <TYPE>.
 *
 * See opal_atomic_cmpset_acq_* for pseudo-code.
 */
#define opal_atomic_cmpset_acq( ADDR, OLDVAL, NEWVAL )           \
   opal_atomic_cmpset_acq_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
                              (int64_t)(NEWVAL), sizeof(*(ADDR)) )


/**
 * Atomic compare and set of pointer with release semantics. This
 * macro detect at compile time the type of the first argument 
 * and choose the correct function to b
 *
 * \note This macro should only be used for integer types.
 *
 * @param addr          Address of <TYPE>.
 * @param oldval        Comparison value <TYPE>.
 * @param newval        New value to set if comparision is true <TYPE>.
 *
 * See opal_atomic_cmpsetrel_* for pseudo-code.
 */
#define opal_atomic_cmpset_rel( ADDR, OLDVAL, NEWVAL )           \
   opal_atomic_cmpset_rel_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
                              (int64_t)(NEWVAL), sizeof(*(ADDR)) )

#endif /* (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) */

#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)

static inline void opal_atomic_add_xx(volatile void* addr, 
                                      int32_t value, size_t length);
static inline void opal_atomic_sub_xx(volatile void* addr, 
                                      int32_t value, size_t length);
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32
static inline int32_t opal_atomic_add_ptr( volatile void* addr, void* delta );
static inline int32_t opal_atomic_sub_ptr( volatile void* addr, void* delta );
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64
static inline int64_t opal_atomic_add_ptr( volatile void* addr, void* delta );
static inline int64_t opal_atomic_sub_ptr( volatile void* addr, void* delta );
#else
#error Atomic arithmetic on pointers not supported
#endif

/**
 * Atomically increment the content depending on the type. This
 * macro detect at compile time the type of the first argument 
 * and choose the correct function to be called.
 *
 * \note This macro should only be used for integer types.
 *
 * @param addr          Address of <TYPE>
 * @param delta         Value to add (converted to <TYPE>).
 */
#define opal_atomic_add( ADDR, VALUE )                                  \
   opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
                       sizeof(*(ADDR)) )

/**
 * Atomically decrement the content depending on the type. This
 * macro detect at compile time the type of the first argument 
 * and choose the correct function to be called.
 *
 * \note This macro should only be used for integer types.
 *
 * @param addr          Address of <TYPE>
 * @param delta         Value to substract (converted to <TYPE>).
 */
#define opal_atomic_sub( ADDR, VALUE )                                  \
   opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE),        \
                      sizeof(*(ADDR)) )

#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */


/**********************************************************************
 *
 * Include system specific inline asm definitions. Otherwise
 * the definitions are in system specific .s files in src/util.
 *
 *********************************************************************/
#include "opal/sys/atomic_impl.h"

END_C_DECLS

#endif /* OPAL_SYS_ATOMIC_H */