src/mm/boehm-gc/libatomic_ops-1.2/src/atomic_ops/generalize.h

   1 /*
   2  * Copyright (c) 2003-2004 Hewlett-Packard Development Company, L.P.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a copy
   5  * of this software and associated documentation files (the "Software"), to deal
   6  * in the Software without restriction, including without limitation the rights
   7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8  * copies of the Software, and to permit persons to whom the Software is
   9  * furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 /*
  24  * Generalize atomic operations for atomic_ops.h.
  25  * Should not be included directly.
  26  *
  27  * We make no attempt to define useless operations, such as
  28  * AO_nop_acquire
  29  * AO_nop_release
  30  *
  31  * We have also so far neglected to define some others, which
  32  * do not appear likely to be useful, e.g. stores with acquire
  33  * or read barriers.
  34  *
  35  * This file is sometimes included twice by atomic_ops.h.
  36  * All definitions include explicit checks that we are not replacing
  37  * an earlier definition.  In general, more desirable expansions
  38  * appear earlier so that we are more likely to use them.
  39  *
  40  * We only make safe generalizations, except that by default we define
  41  * the ...dd_acquire_read operations to be equivalent to those without
  42  * a barrier.  On platforms for which this is unsafe, the platform-specific
  43  * file must define AO_NO_DD_ORDERING.
  44  */
  45
  46 #ifndef ATOMIC_OPS_H
  47 # error Atomic_ops_generalize.h should not be included directly.
  48 #endif
  49
  50 #if AO_CHAR_TS_T
  51 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
  52          AO_char_compare_and_swap_full(a,o,n)
  53 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
  54          AO_char_compare_and_swap_acquire(a,o,n)
  55 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
  56          AO_char_compare_and_swap_release(a,o,n)
  57 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
  58          AO_char_compare_and_swap(a,o,n)
  59 #endif
  60
  61 #if AO_AO_TS_T
  62 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
  63          AO_compare_and_swap_full(a,o,n)
  64 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
  65          AO_compare_and_swap_acquire(a,o,n)
  66 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
  67          AO_compare_and_swap_release(a,o,n)
  68 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
  69          AO_compare_and_swap(a,o,n)
  70 #endif
  71
  72 /* Generate test_and_set_full, if necessary and possible.       */
  73 #if !defined(AO_HAVE_test_and_set) && \
  74     !defined(AO_HAVE_test_and_set_release) && \
  75     !defined(AO_HAVE_test_and_set_acquire) && \
  76     !defined(AO_HAVE_test_and_set_read) && \
  77     !defined(AO_HAVE_test_and_set_full)
  78 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_full) || \
  79       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_full)
  80      AO_INLINE AO_TS_VAL_t
  81      AO_test_and_set_full(volatile AO_TS_t *addr)
  82      {
  83        if (AO_TS_COMPARE_AND_SWAP_FULL(addr, AO_TS_CLEAR, AO_TS_SET))
  84          return AO_TS_CLEAR;
  85        else
  86          return AO_TS_SET;
  87      }
  88 #    define AO_HAVE_test_and_set_full
  89 #  endif /* AO_HAVE_compare_and_swap_full */
  90
  91 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_acquire) || \
  92       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_acquire)
  93      AO_INLINE AO_TS_VAL_t
  94      AO_test_and_set_acquire(volatile AO_TS_t *addr)
  95      {
  96        if (AO_TS_COMPARE_AND_SWAP_ACQUIRE(addr, AO_TS_CLEAR, AO_TS_SET))
  97          return AO_TS_CLEAR;
  98        else
  99          return AO_TS_SET;
 100      }
 101 #    define AO_HAVE_test_and_set_acquire
 102 #  endif /* AO_HAVE_compare_and_swap_acquire */
 103
 104 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_release) || \
 105       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_release)
 106      AO_INLINE AO_TS_VAL_t
 107      AO_test_and_set_release(volatile AO_TS_t *addr)
 108      {
 109        if (AO_TS_COMPARE_AND_SWAP_RELEASE(addr, AO_TS_CLEAR, AO_TS_SET))
 110          return AO_TS_CLEAR;
 111        else
 112          return AO_TS_SET;
 113      }
 114 #    define AO_HAVE_test_and_set_release
 115 #  endif /* AO_HAVE_compare_and_swap_release */
 116
 117 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap) || \
 118       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap)
 119      AO_INLINE AO_TS_VAL_t
 120      AO_test_and_set(volatile AO_TS_t *addr)
 121      {
 122        if (AO_TS_COMPARE_AND_SWAP(addr, AO_TS_CLEAR, AO_TS_SET))
 123          return AO_TS_CLEAR;
 124        else
 125          return AO_TS_SET;
 126      }
 127 #    define AO_HAVE_test_and_set
 128 #  endif /* AO_HAVE_compare_and_swap */
 129
 130 #  if defined(AO_HAVE_test_and_set) && defined(AO_HAVE_nop_full) \
 131       && !defined(AO_HAVE_test_and_set_acquire)
 132      AO_INLINE AO_TS_VAL_t
 133      AO_test_and_set_acquire(volatile AO_TS_t *addr)
 134      {
 135        AO_TS_VAL_t result = AO_test_and_set(addr);
 136        AO_nop_full();
 137        return result;
 138      }
 139 #    define AO_HAVE_test_and_set_acquire
 140 #  endif
 141
 142 #endif /* No prior test and set */
 143
 144 /* Nop */
 145 #if !defined(AO_HAVE_nop)
 146    AO_INLINE void AO_nop(void) {}
 147 #  define AO_HAVE_nop
 148 #endif
 149
 150 #if defined(AO_HAVE_test_and_set_full) && !defined(AO_HAVE_nop_full)
 151    AO_INLINE void
 152    AO_nop_full(void)
 153    {
 154      AO_TS_t dummy = AO_TS_INITIALIZER;
 155      AO_test_and_set_full(&dummy);
 156    }
 157 #  define AO_HAVE_nop_full
 158 #endif
 159
 160 #if defined(AO_HAVE_nop_acquire)
 161 #  error AO_nop_acquire is useless: dont define.
 162 #endif
 163 #if defined(AO_HAVE_nop_release)
 164 #  error AO_nop_release is useless: dont define.
 165 #endif
 166
 167 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_read)
 168 #  define AO_nop_read() AO_nop_full()
 169 #  define AO_HAVE_nop_read
 170 #endif
 171
 172 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_write)
 173 #  define AO_nop_write() AO_nop_full()
 174 #  define AO_HAVE_nop_write
 175 #endif
 176
 177 /* Load */
 178 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
 179 #  define AO_load_acquire(addr) AO_load_full(addr)
 180 #  define AO_HAVE_load_acquire
 181 #endif
 182
 183 #if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
 184 #  define AO_load(addr) AO_load_acquire(addr)
 185 #  define AO_HAVE_load
 186 #endif
 187
 188 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
 189 #  define AO_load_read(addr) AO_load_full(addr)
 190 #  define AO_HAVE_load_read
 191 #endif
 192
 193 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_acquire)
 194 #  define AO_load_acquire_read(addr) AO_load_acquire(addr)
 195 #  define AO_HAVE_load_acquire_read
 196 #endif
 197
 198 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) && \
 199     !defined(AO_HAVE_load_acquire)
 200    AO_INLINE AO_t
 201    AO_load_acquire(const volatile AO_t *addr)
 202    {
 203      AO_t result = AO_load(addr);
 204      /* Acquire barrier would be useless, since the load could be delayed  */
 205      /* beyond it.                                                         */
 206      AO_nop_full();
 207      return result;
 208    }
 209 #  define AO_HAVE_load_acquire
 210 #endif
 211
 212 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) && \
 213     !defined(AO_HAVE_load_read)
 214    AO_INLINE AO_t
 215    AO_load_read(const volatile AO_t *addr)
 216    {
 217      AO_t result = AO_load(addr);
 218      /* Acquire barrier would be useless, since the load could be delayed  */
 219      /* beyond it.                                                         */
 220      AO_nop_read();
 221      return result;
 222    }
 223 #  define AO_HAVE_load_read
 224 #endif
 225
 226 #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) && \
 227     !defined(AO_HAVE_load_full)
 228 #  define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
 229 #  define AO_HAVE_load_full
 230 #endif
 231
 232 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_read)
 233 #  define AO_load_acquire_read(addr) AO_load_read(addr)
 234 #  define AO_HAVE_load_acquire_read
 235 #endif
 236
 237 #if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
 238 #  define AO_load(addr) AO_load_acquire_read(addr)
 239 #  define AO_HAVE_load
 240 #endif
 241
 242 #ifdef AO_NO_DD_ORDERING
 243 #  if defined(AO_HAVE_load_acquire_read)
 244 #    define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
 245 #    define AO_HAVE_load_dd_acquire_read
 246 #  endif
 247 #else
 248 #  if defined(AO_HAVE_load)
 249 #    define AO_load_dd_acquire_read(addr) AO_load(addr)
 250 #    define AO_HAVE_load_dd_acquire_read
 251 #  endif
 252 #endif
 253
 254
 255 /* Store */
 256
 257 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
 258 #  define AO_store_release(addr,val) AO_store_full(addr,val)
 259 #  define AO_HAVE_store_release
 260 #endif
 261
 262 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
 263 #  define AO_store(addr, val) AO_store_release(addr,val)
 264 #  define AO_HAVE_store
 265 #endif
 266
 267 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
 268 #  define AO_store_write(addr,val) AO_store_full(addr,val)
 269 #  define AO_HAVE_store_write
 270 #endif
 271
 272 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store_release_write)
 273 #  define AO_store_release_write(addr, val) AO_store_release(addr,val)
 274 #  define AO_HAVE_store_release_write
 275 #endif
 276
 277 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
 278 #  define AO_store(addr, val) AO_store_write(addr,val)
 279 #  define AO_HAVE_store
 280 #endif
 281
 282 #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) && \
 283     !defined(AO_HAVE_store_release)
 284 #  define AO_store_release(addr,val) (AO_nop_full(), AO_store(addr,val))
 285 #  define AO_HAVE_store_release
 286 #endif
 287
 288 #if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) && \
 289      !defined(AO_HAVE_store_write)
 290 #  define AO_store_write(addr, val) (AO_nop_write(), AO_store(addr,val))
 291 #  define AO_HAVE_store_write
 292 #endif
 293
 294 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store_release_write)
 295 #  define AO_store_release_write(addr, val) AO_store_write(addr,val)
 296 #  define AO_HAVE_store_release_write
 297 #endif
 298
 299 #if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) && \
 300     !defined(AO_HAVE_store_full)
 301 #  define AO_store_full(addr, val) (AO_store_release(addr, val), AO_nop_full())
 302 #  define AO_HAVE_store_full
 303 #endif
 304
 305 /* NEC LE-IT: Test and set */
 306 #if defined(AO_HAVE_test_and_set) && \
 307         defined(AO_HAVE_nop_full) && \
 308     !defined(AO_HAVE_test_and_set_release)
 309 #       define AO_test_and_set_release(addr) \
 310         (AO_nop_full(), AO_test_and_set(addr))
 311 #  define AO_HAVE_test_and_set_release
 312 #endif
 313
 314 #if defined(AO_HAVE_test_and_set) && \
 315         defined(AO_HAVE_nop_full) && \
 316     !defined(AO_HAVE_test_and_set_acquire)
 317 AO_INLINE AO_TS_t
 318 AO_test_and_set_acquire(volatile AO_TS_t *addr)
 319 {
 320         AO_TS_t res = AO_test_and_set(addr);
 321         AO_nop_full();
 322         return res;
 323 }
 324 #  define AO_HAVE_test_and_set_acquire
 325 #endif
 326
 327
 328 /* Fetch_and_add */
 329 /* We first try to implement fetch_and_add variants in terms    */
 330 /* of the corresponding compare_and_swap variants to minimize   */
 331 /* adding barriers.                                             */
 332 #if defined(AO_HAVE_compare_and_swap_full) && \
 333     !defined(AO_HAVE_fetch_and_add_full)
 334    AO_INLINE AO_t
 335    AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
 336    {
 337      AO_t old;
 338      do
 339        {
 340          old = *addr;
 341        }
 342      while (!AO_compare_and_swap_full(addr, old, old+incr));
 343      return old;
 344    }
 345 #  define AO_HAVE_fetch_and_add_full
 346 #endif
 347
 348 #if defined(AO_HAVE_compare_and_swap_acquire) && \
 349     !defined(AO_HAVE_fetch_and_add_acquire)
 350    AO_INLINE AO_t
 351    AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
 352    {
 353      AO_t old;
 354      do
 355        {
 356          old = *addr;
 357        }
 358      while (!AO_compare_and_swap_acquire(addr, old, old+incr));
 359      return old;
 360    }
 361 #  define AO_HAVE_fetch_and_add_acquire
 362 #endif
 363
 364 #if defined(AO_HAVE_compare_and_swap_release) && \
 365     !defined(AO_HAVE_fetch_and_add_release)
 366    AO_INLINE AO_t
 367    AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
 368    {
 369      AO_t old;
 370      do
 371        {
 372          old = *addr;
 373        }
 374      while (!AO_compare_and_swap_release(addr, old, old+incr));
 375      return old;
 376    }
 377 #  define AO_HAVE_fetch_and_add_release
 378 #endif
 379
 380 #if defined(AO_HAVE_compare_and_swap) && \
 381     !defined(AO_HAVE_fetch_and_add)
 382    AO_INLINE AO_t
 383    AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
 384    {
 385      AO_t old;
 386      do
 387        {
 388          old = *addr;
 389        }
 390      while (!AO_compare_and_swap(addr, old, old+incr));
 391      return old;
 392    }
 393 #  define AO_HAVE_fetch_and_add
 394 #endif
 395
 396 #if defined(AO_HAVE_fetch_and_add_full)
 397 #  if !defined(AO_HAVE_fetch_and_add_release)
 398 #    define AO_fetch_and_add_release(addr, val) \
 399          AO_fetch_and_add_full(addr, val)
 400 #    define AO_HAVE_fetch_and_add_release
 401 #  endif
 402 #  if !defined(AO_HAVE_fetch_and_add_acquire)
 403 #    define AO_fetch_and_add_acquire(addr, val) \
 404          AO_fetch_and_add_full(addr, val)
 405 #    define AO_HAVE_fetch_and_add_acquire
 406 #  endif
 407 #  if !defined(AO_HAVE_fetch_and_add_write)
 408 #    define AO_fetch_and_add_write(addr, val) \
 409          AO_fetch_and_add_full(addr, val)
 410 #    define AO_HAVE_fetch_and_add_write
 411 #  endif
 412 #  if !defined(AO_HAVE_fetch_and_add_read)
 413 #    define AO_fetch_and_add_read(addr, val) \
 414          AO_fetch_and_add_full(addr, val)
 415 #    define AO_HAVE_fetch_and_add_read
 416 #  endif
 417 #endif /* AO_HAVE_fetch_and_add_full */
 418
 419 #if !defined(AO_HAVE_fetch_and_add) && \
 420     defined(AO_HAVE_fetch_and_add_release)
 421 #  define AO_fetch_and_add(addr, val) \
 422         AO_fetch_and_add_release(addr, val)
 423 #  define AO_HAVE_fetch_and_add
 424 #endif
 425 #if !defined(AO_HAVE_fetch_and_add) && \
 426     defined(AO_HAVE_fetch_and_add_acquire)
 427 #  define AO_fetch_and_add(addr, val) \
 428         AO_fetch_and_add_acquire(addr, val)
 429 #  define AO_HAVE_fetch_and_add
 430 #endif
 431 #if !defined(AO_HAVE_fetch_and_add) && \
 432     defined(AO_HAVE_fetch_and_add_write)
 433 #  define AO_fetch_and_add(addr, val) \
 434         AO_fetch_and_add_write(addr, val)
 435 #  define AO_HAVE_fetch_and_add
 436 #endif
 437 #if !defined(AO_HAVE_fetch_and_add) && \
 438     defined(AO_HAVE_fetch_and_add_read)
 439 #  define AO_fetch_and_add(addr, val) \
 440         AO_fetch_and_add_read(addr, val)
 441 #  define AO_HAVE_fetch_and_add
 442 #endif
 443
 444 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
 445     defined(AO_HAVE_nop_full) && \
 446     !defined(AO_HAVE_fetch_and_add_full)
 447 #  define AO_fetch_and_add_full(addr, val) \
 448         (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
 449 #  define AO_HAVE_fetch_and_add_full
 450 #endif
 451
 452 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
 453     defined(AO_HAVE_fetch_and_add_write)
 454 #  define AO_fetch_and_add_release_write(addr, val) \
 455         AO_fetch_and_add_write(addr, val)
 456 #  define AO_HAVE_fetch_and_add_release_write
 457 #endif
 458 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
 459     defined(AO_HAVE_fetch_and_add_release)
 460 #  define AO_fetch_and_add_release_write(addr, val) \
 461         AO_fetch_and_add_release(addr, val)
 462 #  define AO_HAVE_fetch_and_add_release_write
 463 #endif
 464 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
 465     defined(AO_HAVE_fetch_and_add_read)
 466 #  define AO_fetch_and_add_acquire_read(addr, val) \
 467         AO_fetch_and_add_read(addr, val)
 468 #  define AO_HAVE_fetch_and_add_acquire_read
 469 #endif
 470 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
 471     defined(AO_HAVE_fetch_and_add_acquire)
 472 #  define AO_fetch_and_add_acquire_read(addr, val) \
 473         AO_fetch_and_add_acquire(addr, val)
 474 #  define AO_HAVE_fetch_and_add_acquire_read
 475 #endif
 476
 477 #ifdef AO_NO_DD_ORDERING
 478 #  if defined(AO_HAVE_fetch_and_add_acquire_read)
 479 #    define AO_fetch_and_add_dd_acquire_read(addr, val) \
 480         AO_fetch_and_add_acquire_read(addr, val)
 481 #    define AO_HAVE_fetch_and_add_dd_acquire_read
 482 #  endif
 483 #else
 484 #  if defined(AO_HAVE_fetch_and_add)
 485 #    define AO_fetch_and_add_dd_acquire_read(addr, val) \
 486         AO_fetch_and_add(addr, val)
 487 #    define AO_HAVE_fetch_and_add_dd_acquire_read
 488 #  endif
 489 #endif
 490
 491 /* Fetch_and_add1 */
 492
 493 #if defined(AO_HAVE_fetch_and_add_full) &&\
 494     !defined(AO_HAVE_fetch_and_add1_full)
 495 #  define AO_fetch_and_add1_full(addr) AO_fetch_and_add_full(addr,1)
 496 #  define AO_HAVE_fetch_and_add1_full
 497 #endif
 498 #if defined(AO_HAVE_fetch_and_add_release) &&\
 499     !defined(AO_HAVE_fetch_and_add1_release)
 500 #  define AO_fetch_and_add1_release(addr) AO_fetch_and_add_release(addr,1)
 501 #  define AO_HAVE_fetch_and_add1_release
 502 #endif
 503 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
 504     !defined(AO_HAVE_fetch_and_add1_acquire)
 505 #  define AO_fetch_and_add1_acquire(addr) AO_fetch_and_add_acquire(addr,1)
 506 #  define AO_HAVE_fetch_and_add1_acquire
 507 #endif
 508 #if defined(AO_HAVE_fetch_and_add_write) &&\
 509     !defined(AO_HAVE_fetch_and_add1_write)
 510 #  define AO_fetch_and_add1_write(addr) AO_fetch_and_add_write(addr,1)
 511 #  define AO_HAVE_fetch_and_add1_write
 512 #endif
 513 #if defined(AO_HAVE_fetch_and_add_read) &&\
 514     !defined(AO_HAVE_fetch_and_add1_read)
 515 #  define AO_fetch_and_add1_read(addr) AO_fetch_and_add_read(addr,1)
 516 #  define AO_HAVE_fetch_and_add1_read
 517 #endif
 518 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
 519     !defined(AO_HAVE_fetch_and_add1_release_write)
 520 #  define AO_fetch_and_add1_release_write(addr) \
 521         AO_fetch_and_add_release_write(addr,1)
 522 #  define AO_HAVE_fetch_and_add1_release_write
 523 #endif
 524 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
 525     !defined(AO_HAVE_fetch_and_add1_acquire_read)
 526 #  define AO_fetch_and_add1_acquire_read(addr) \
 527         AO_fetch_and_add_acquire_read(addr,1)
 528 #  define AO_HAVE_fetch_and_add1_acquire_read
 529 #endif
 530 #if defined(AO_HAVE_fetch_and_add) &&\
 531     !defined(AO_HAVE_fetch_and_add1)
 532 #  define AO_fetch_and_add1(addr) \
 533         AO_fetch_and_add(addr,1)
 534 #  define AO_HAVE_fetch_and_add1
 535 #endif
 536
 537 #if defined(AO_HAVE_fetch_and_add1_full)
 538 #  if !defined(AO_HAVE_fetch_and_add1_release)
 539 #    define AO_fetch_and_add1_release(addr) \
 540          AO_fetch_and_add1_full(addr)
 541 #    define AO_HAVE_fetch_and_add1_release
 542 #  endif
 543 #  if !defined(AO_HAVE_fetch_and_add1_acquire)
 544 #    define AO_fetch_and_add1_acquire(addr) \
 545          AO_fetch_and_add1_full(addr)
 546 #    define AO_HAVE_fetch_and_add1_acquire
 547 #  endif
 548 #  if !defined(AO_HAVE_fetch_and_add1_write)
 549 #    define AO_fetch_and_add1_write(addr) \
 550          AO_fetch_and_add1_full(addr)
 551 #    define AO_HAVE_fetch_and_add1_write
 552 #  endif
 553 #  if !defined(AO_HAVE_fetch_and_add1_read)
 554 #    define AO_fetch_and_add1_read(addr) \
 555          AO_fetch_and_add1_full(addr)
 556 #    define AO_HAVE_fetch_and_add1_read
 557 #  endif
 558 #endif /* AO_HAVE_fetch_and_add1_full */
 559
 560 #if !defined(AO_HAVE_fetch_and_add1) && \
 561     defined(AO_HAVE_fetch_and_add1_release)
 562 #  define AO_fetch_and_add1(addr) \
 563         AO_fetch_and_add1_release(addr)
 564 #  define AO_HAVE_fetch_and_add1
 565 #endif
 566 #if !defined(AO_HAVE_fetch_and_add1) && \
 567     defined(AO_HAVE_fetch_and_add1_acquire)
 568 #  define AO_fetch_and_add1(addr) \
 569         AO_fetch_and_add1_acquire(addr)
 570 #  define AO_HAVE_fetch_and_add1
 571 #endif
 572 #if !defined(AO_HAVE_fetch_and_add1) && \
 573     defined(AO_HAVE_fetch_and_add1_write)
 574 #  define AO_fetch_and_add1(addr) \
 575         AO_fetch_and_add1_write(addr)
 576 #  define AO_HAVE_fetch_and_add1
 577 #endif
 578 #if !defined(AO_HAVE_fetch_and_add1) && \
 579     defined(AO_HAVE_fetch_and_add1_read)
 580 #  define AO_fetch_and_add1(addr) \
 581         AO_fetch_and_add1_read(addr)
 582 #  define AO_HAVE_fetch_and_add1
 583 #endif
 584
 585 #if defined(AO_HAVE_fetch_and_add1_acquire) &&\
 586     defined(AO_HAVE_nop_full) && \
 587     !defined(AO_HAVE_fetch_and_add1_full)
 588 #  define AO_fetch_and_add1_full(addr) \
 589         (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
 590 #  define AO_HAVE_fetch_and_add1_full
 591 #endif
 592
 593 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
 594     defined(AO_HAVE_fetch_and_add1_write)
 595 #  define AO_fetch_and_add1_release_write(addr) \
 596         AO_fetch_and_add1_write(addr)
 597 #  define AO_HAVE_fetch_and_add1_release_write
 598 #endif
 599 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
 600     defined(AO_HAVE_fetch_and_add1_release)
 601 #  define AO_fetch_and_add1_release_write(addr) \
 602         AO_fetch_and_add1_release(addr)
 603 #  define AO_HAVE_fetch_and_add1_release_write
 604 #endif
 605 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
 606     defined(AO_HAVE_fetch_and_add1_read)
 607 #  define AO_fetch_and_add1_acquire_read(addr) \
 608         AO_fetch_and_add1_read(addr)
 609 #  define AO_HAVE_fetch_and_add1_acquire_read
 610 #endif
 611 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
 612     defined(AO_HAVE_fetch_and_add1_acquire)
 613 #  define AO_fetch_and_add1_acquire_read(addr) \
 614         AO_fetch_and_add1_acquire(addr)
 615 #  define AO_HAVE_fetch_and_add1_acquire_read
 616 #endif
 617
 618 #ifdef AO_NO_DD_ORDERING
 619 #  if defined(AO_HAVE_fetch_and_add1_acquire_read)
 620 #    define AO_fetch_and_add1_dd_acquire_read(addr) \
 621         AO_fetch_and_add1_acquire_read(addr)
 622 #    define AO_HAVE_fetch_and_add1_dd_acquire_read
 623 #  endif
 624 #else
 625 #  if defined(AO_HAVE_fetch_and_add1)
 626 #    define AO_fetch_and_add1_dd_acquire_read(addr) AO_fetch_and_add1(addr)
 627 #    define AO_HAVE_fetch_and_add1_dd_acquire_read
 628 #  endif
 629 #endif
 630
 631 /* Fetch_and_sub1 */
 632
 633 #if defined(AO_HAVE_fetch_and_add_full) &&\
 634     !defined(AO_HAVE_fetch_and_sub1_full)
 635 #  define AO_fetch_and_sub1_full(addr) AO_fetch_and_add_full(addr,(AO_t)(-1))
 636 #  define AO_HAVE_fetch_and_sub1_full
 637 #endif
 638 #if defined(AO_HAVE_fetch_and_add_release) &&\
 639     !defined(AO_HAVE_fetch_and_sub1_release)
 640 #  define AO_fetch_and_sub1_release(addr) \
 641         AO_fetch_and_add_release(addr,(AO_t)(-1))
 642 #  define AO_HAVE_fetch_and_sub1_release
 643 #endif
 644 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
 645     !defined(AO_HAVE_fetch_and_sub1_acquire)
 646 #  define AO_fetch_and_sub1_acquire(addr) \
 647         AO_fetch_and_add_acquire(addr,(AO_t)(-1))
 648 #  define AO_HAVE_fetch_and_sub1_acquire
 649 #endif
 650 #if defined(AO_HAVE_fetch_and_add_write) &&\
 651     !defined(AO_HAVE_fetch_and_sub1_write)
 652 #  define AO_fetch_and_sub1_write(addr) \
 653         AO_fetch_and_add_write(addr,(AO_t)(-1))
 654 #  define AO_HAVE_fetch_and_sub1_write
 655 #endif
 656 #if defined(AO_HAVE_fetch_and_add_read) &&\
 657     !defined(AO_HAVE_fetch_and_sub1_read)
 658 #  define AO_fetch_and_sub1_read(addr) \
 659         AO_fetch_and_add_read(addr,(AO_t)(-1))
 660 #  define AO_HAVE_fetch_and_sub1_read
 661 #endif
 662 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
 663     !defined(AO_HAVE_fetch_and_sub1_release_write)
 664 #  define AO_fetch_and_sub1_release_write(addr) \
 665         AO_fetch_and_add_release_write(addr,(AO_t)(-1))
 666 #  define AO_HAVE_fetch_and_sub1_release_write
 667 #endif
 668 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
 669     !defined(AO_HAVE_fetch_and_sub1_acquire_read)
 670 #  define AO_fetch_and_sub1_acquire_read(addr) \
 671         AO_fetch_and_add_acquire_read(addr,(AO_t)(-1))
 672 #  define AO_HAVE_fetch_and_sub1_acquire_read
 673 #endif
 674 #if defined(AO_HAVE_fetch_and_add) &&\
 675     !defined(AO_HAVE_fetch_and_sub1)
 676 #  define AO_fetch_and_sub1(addr) \
 677         AO_fetch_and_add(addr,(AO_t)(-1))
 678 #  define AO_HAVE_fetch_and_sub1
 679 #endif
 680
 681 #if defined(AO_HAVE_fetch_and_sub1_full)
 682 #  if !defined(AO_HAVE_fetch_and_sub1_release)
 683 #    define AO_fetch_and_sub1_release(addr) \
 684          AO_fetch_and_sub1_full(addr)
 685 #    define AO_HAVE_fetch_and_sub1_release
 686 #  endif
 687 #  if !defined(AO_HAVE_fetch_and_sub1_acquire)
 688 #    define AO_fetch_and_sub1_acquire(addr) \
 689          AO_fetch_and_sub1_full(addr)
 690 #    define AO_HAVE_fetch_and_sub1_acquire
 691 #  endif
 692 #  if !defined(AO_HAVE_fetch_and_sub1_write)
 693 #    define AO_fetch_and_sub1_write(addr) \
 694          AO_fetch_and_sub1_full(addr)
 695 #    define AO_HAVE_fetch_and_sub1_write
 696 #  endif
 697 #  if !defined(AO_HAVE_fetch_and_sub1_read)
 698 #    define AO_fetch_and_sub1_read(addr) \
 699          AO_fetch_and_sub1_full(addr)
 700 #    define AO_HAVE_fetch_and_sub1_read
 701 #  endif
 702 #endif /* AO_HAVE_fetch_and_sub1_full */
 703
 704 #if !defined(AO_HAVE_fetch_and_sub1) && \
 705     defined(AO_HAVE_fetch_and_sub1_release)
 706 #  define AO_fetch_and_sub1(addr) \
 707         AO_fetch_and_sub1_release(addr)
 708 #  define AO_HAVE_fetch_and_sub1
 709 #endif
 710 #if !defined(AO_HAVE_fetch_and_sub1) && \
 711     defined(AO_HAVE_fetch_and_sub1_acquire)
 712 #  define AO_fetch_and_sub1(addr) \
 713         AO_fetch_and_sub1_acquire(addr)
 714 #  define AO_HAVE_fetch_and_sub1
 715 #endif
 716 #if !defined(AO_HAVE_fetch_and_sub1) && \
 717     defined(AO_HAVE_fetch_and_sub1_write)
 718 #  define AO_fetch_and_sub1(addr) \
 719         AO_fetch_and_sub1_write(addr)
 720 #  define AO_HAVE_fetch_and_sub1
 721 #endif
 722 #if !defined(AO_HAVE_fetch_and_sub1) && \
 723     defined(AO_HAVE_fetch_and_sub1_read)
 724 #  define AO_fetch_and_sub1(addr) \
 725         AO_fetch_and_sub1_read(addr)
 726 #  define AO_HAVE_fetch_and_sub1
 727 #endif
 728
 729 #if defined(AO_HAVE_fetch_and_sub1_acquire) &&\
 730     defined(AO_HAVE_nop_full) && \
 731     !defined(AO_HAVE_fetch_and_sub1_full)
 732 #  define AO_fetch_and_sub1_full(addr) \
 733         (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
 734 #  define AO_HAVE_fetch_and_sub1_full
 735 #endif
 736
 737 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
 738     defined(AO_HAVE_fetch_and_sub1_write)
 739 #  define AO_fetch_and_sub1_release_write(addr) \
 740         AO_fetch_and_sub1_write(addr)
 741 #  define AO_HAVE_fetch_and_sub1_release_write
 742 #endif
 743 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
 744     defined(AO_HAVE_fetch_and_sub1_release)
 745 #  define AO_fetch_and_sub1_release_write(addr) \
 746         AO_fetch_and_sub1_release(addr)
 747 #  define AO_HAVE_fetch_and_sub1_release_write
 748 #endif
 749 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
 750     defined(AO_HAVE_fetch_and_sub1_read)
 751 #  define AO_fetch_and_sub1_acquire_read(addr) \
 752         AO_fetch_and_sub1_read(addr)
 753 #  define AO_HAVE_fetch_and_sub1_acquire_read
 754 #endif
 755 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
 756     defined(AO_HAVE_fetch_and_sub1_acquire)
 757 #  define AO_fetch_and_sub1_acquire_read(addr) \
 758         AO_fetch_and_sub1_acquire(addr)
 759 #  define AO_HAVE_fetch_and_sub1_acquire_read
 760 #endif
 761
 762 #ifdef AO_NO_DD_ORDERING
 763 #  if defined(AO_HAVE_fetch_and_sub1_acquire_read)
 764 #    define AO_fetch_and_sub1_dd_acquire_read(addr) \
 765         AO_fetch_and_sub1_acquire_read(addr)
 766 #    define AO_HAVE_fetch_and_sub1_dd_acquire_read
 767 #  endif
 768 #else
 769 #  if defined(AO_HAVE_fetch_and_sub1)
 770 #    define AO_fetch_and_sub1_dd_acquire_read(addr) AO_fetch_and_sub1(addr)
 771 #    define AO_HAVE_fetch_and_sub1_dd_acquire_read
 772 #  endif
 773 #endif
 774
 775 /* Atomic or */
 776 #if defined(AO_HAVE_compare_and_swap_full) && \
 777     !defined(AO_HAVE_or_full)
 778    AO_INLINE void
 779    AO_or_full(volatile AO_t *addr, AO_t incr)
 780    {
 781      AO_t old;
 782      do
 783        {
 784          old = *addr;
 785        }
 786      while (!AO_compare_and_swap_full(addr, old, (old | incr)));
 787    }
 788 #  define AO_HAVE_or_full
 789 #endif
 790
 791 #if defined(AO_HAVE_or_full)
 792 #  if !defined(AO_HAVE_or_release)
 793 #    define AO_or_release(addr, val) \
 794          AO_or_full(addr, val)
 795 #    define AO_HAVE_or_release
 796 #  endif
 797 #  if !defined(AO_HAVE_or_acquire)
 798 #    define AO_or_acquire(addr, val) \
 799          AO_or_full(addr, val)
 800 #    define AO_HAVE_or_acquire
 801 #  endif
 802 #  if !defined(AO_HAVE_or_write)
 803 #    define AO_or_write(addr, val) \
 804          AO_or_full(addr, val)
 805 #    define AO_HAVE_or_write
 806 #  endif
 807 #  if !defined(AO_HAVE_or_read)
 808 #    define AO_or_read(addr, val) \
 809          AO_or_full(addr, val)
 810 #    define AO_HAVE_or_read
 811 #  endif
 812 #endif /* AO_HAVE_or_full */
 813
 814 #if !defined(AO_HAVE_or) && \
 815     defined(AO_HAVE_or_release)
 816 #  define AO_or(addr, val) \
 817         AO_or_release(addr, val)
 818 #  define AO_HAVE_or
 819 #endif
 820 #if !defined(AO_HAVE_or) && \
 821     defined(AO_HAVE_or_acquire)
 822 #  define AO_or(addr, val) \
 823         AO_or_acquire(addr, val)
 824 #  define AO_HAVE_or
 825 #endif
 826 #if !defined(AO_HAVE_or) && \
 827     defined(AO_HAVE_or_write)
 828 #  define AO_or(addr, val) \
 829         AO_or_write(addr, val)
 830 #  define AO_HAVE_or
 831 #endif
 832 #if !defined(AO_HAVE_or) && \
 833     defined(AO_HAVE_or_read)
 834 #  define AO_or(addr, val) \
 835         AO_or_read(addr, val)
 836 #  define AO_HAVE_or
 837 #endif
 838
 839 #if defined(AO_HAVE_or_acquire) &&\
 840     defined(AO_HAVE_nop_full) && \
 841     !defined(AO_HAVE_or_full)
 842 #  define AO_or_full(addr, val) \
 843         (AO_nop_full(), AO_or_acquire(addr, val))
 844 #endif
 845
 846 #if !defined(AO_HAVE_or_release_write) && \
 847     defined(AO_HAVE_or_write)
 848 #  define AO_or_release_write(addr, val) \
 849         AO_or_write(addr, val)
 850 #  define AO_HAVE_or_release_write
 851 #endif
 852 #if !defined(AO_HAVE_or_release_write) && \
 853     defined(AO_HAVE_or_release)
 854 #  define AO_or_release_write(addr, val) \
 855         AO_or_release(addr, val)
 856 #  define AO_HAVE_or_release_write
 857 #endif
 858 #if !defined(AO_HAVE_or_acquire_read) && \
 859     defined(AO_HAVE_or_read)
 860 #  define AO_or_acquire_read(addr, val) \
 861         AO_or_read(addr, val)
 862 #  define AO_HAVE_or_acquire_read
 863 #endif
 864 #if !defined(AO_HAVE_or_acquire_read) && \
 865     defined(AO_HAVE_or_acquire)
 866 #  define AO_or_acquire_read(addr, val) \
 867         AO_or_acquire(addr, val)
 868 #  define AO_HAVE_or_acquire_read
 869 #endif
 870
 871 /* dd_aquire_read is meaningless.       */
 872
 873 /* Test_and_set */
 874
 875 #if defined(AO_HAVE_test_and_set_full)
 876 #  if !defined(AO_HAVE_test_and_set_release)
 877 #    define AO_test_and_set_release(addr) \
 878          AO_test_and_set_full(addr)
 879 #    define AO_HAVE_test_and_set_release
 880 #  endif
 881 #  if !defined(AO_HAVE_test_and_set_acquire)
 882 #    define AO_test_and_set_acquire(addr) \
 883          AO_test_and_set_full(addr)
 884 #    define AO_HAVE_test_and_set_acquire
 885 #  endif
 886 #  if !defined(AO_HAVE_test_and_set_write)
 887 #    define AO_test_and_set_write(addr) \
 888          AO_test_and_set_full(addr)
 889 #    define AO_HAVE_test_and_set_write
 890 #  endif
 891 #  if !defined(AO_HAVE_test_and_set_read)
 892 #    define AO_test_and_set_read(addr) \
 893          AO_test_and_set_full(addr)
 894 #    define AO_HAVE_test_and_set_read
 895 #  endif
 896 #endif /* AO_HAVE_test_and_set_full */
 897
 898 #if !defined(AO_HAVE_test_and_set) && \
 899     defined(AO_HAVE_test_and_set_release)
 900 #  define AO_test_and_set(addr) \
 901         AO_test_and_set_release(addr)
 902 #  define AO_HAVE_test_and_set
 903 #endif
 904 #if !defined(AO_HAVE_test_and_set) && \
 905     defined(AO_HAVE_test_and_set_acquire)
 906 #  define AO_test_and_set(addr) \
 907         AO_test_and_set_acquire(addr)
 908 #  define AO_HAVE_test_and_set
 909 #endif
 910 #if !defined(AO_HAVE_test_and_set) && \
 911     defined(AO_HAVE_test_and_set_write)
 912 #  define AO_test_and_set(addr) \
 913         AO_test_and_set_write(addr)
 914 #  define AO_HAVE_test_and_set
 915 #endif
 916 #if !defined(AO_HAVE_test_and_set) && \
 917     defined(AO_HAVE_test_and_set_read)
 918 #  define AO_test_and_set(addr) \
 919         AO_test_and_set_read(addr)
 920 #  define AO_HAVE_test_and_set
 921 #endif
 922
 923 #if defined(AO_HAVE_test_and_set_acquire) &&\
 924     defined(AO_HAVE_nop_full) && \
 925     !defined(AO_HAVE_test_and_set_full)
 926 #  define AO_test_and_set_full(addr) \
 927         (AO_nop_full(), AO_test_and_set_acquire(addr))
 928 #  define AO_HAVE_test_and_set_full
 929 #endif
 930
 931 #if !defined(AO_HAVE_test_and_set_release_write) && \
 932     defined(AO_HAVE_test_and_set_write)
 933 #  define AO_test_and_set_release_write(addr) \
 934         AO_test_and_set_write(addr)
 935 #  define AO_HAVE_test_and_set_release_write
 936 #endif
 937 #if !defined(AO_HAVE_test_and_set_release_write) && \
 938     defined(AO_HAVE_test_and_set_release)
 939 #  define AO_test_and_set_release_write(addr) \
 940         AO_test_and_set_release(addr)
 941 #  define AO_HAVE_test_and_set_release_write
 942 #endif
 943 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
 944     defined(AO_HAVE_test_and_set_read)
 945 #  define AO_test_and_set_acquire_read(addr) \
 946         AO_test_and_set_read(addr)
 947 #  define AO_HAVE_test_and_set_acquire_read
 948 #endif
 949 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
 950     defined(AO_HAVE_test_and_set_acquire)
 951 #  define AO_test_and_set_acquire_read(addr) \
 952         AO_test_and_set_acquire(addr)
 953 #  define AO_HAVE_test_and_set_acquire_read
 954 #endif
 955
 956 #ifdef AO_NO_DD_ORDERING
 957 #  if defined(AO_HAVE_test_and_set_acquire_read)
 958 #    define AO_test_and_set_dd_acquire_read(addr) \
 959         AO_test_and_set_acquire_read(addr)
 960 #    define AO_HAVE_test_and_set_dd_acquire_read
 961 #  endif
 962 #else
 963 #  if defined(AO_HAVE_test_and_set)
 964 #    define AO_test_and_set_dd_acquire_read(addr) AO_test_and_set(addr)
 965 #    define AO_HAVE_test_and_set_dd_acquire_read
 966 #  endif
 967 #endif
 968
 969 /* Compare_and_swap */
 970 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
 971     && !defined(AO_HAVE_compare_and_swap_acquire)
 972    AO_INLINE int
 973    AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
 974    {
 975      int result = AO_compare_and_swap(addr, old, new_val);
 976      AO_nop_full();
 977      return result;
 978    }
 979 #  define AO_HAVE_compare_and_swap_acquire
 980 #endif
 981 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
 982     && !defined(AO_HAVE_compare_and_swap_release)
 983 #  define AO_compare_and_swap_release(addr, old, new_val) \
 984         (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
 985 #  define AO_HAVE_compare_and_swap_release
 986 #endif
 987 #if defined(AO_HAVE_compare_and_swap_full)
 988 #  if !defined(AO_HAVE_compare_and_swap_release)
 989 #    define AO_compare_and_swap_release(addr, old, new_val) \
 990          AO_compare_and_swap_full(addr, old, new_val)
 991 #    define AO_HAVE_compare_and_swap_release
 992 #  endif
 993 #  if !defined(AO_HAVE_compare_and_swap_acquire)
 994 #    define AO_compare_and_swap_acquire(addr, old, new_val) \
 995          AO_compare_and_swap_full(addr, old, new_val)
 996 #    define AO_HAVE_compare_and_swap_acquire
 997 #  endif
 998 #  if !defined(AO_HAVE_compare_and_swap_write)
 999 #    define AO_compare_and_swap_write(addr, old, new_val) \
1000          AO_compare_and_swap_full(addr, old, new_val)
1001 #    define AO_HAVE_compare_and_swap_write
1002 #  endif
1003 #  if !defined(AO_HAVE_compare_and_swap_read)
1004 #    define AO_compare_and_swap_read(addr, old, new_val) \
1005          AO_compare_and_swap_full(addr, old, new_val)
1006 #    define AO_HAVE_compare_and_swap_read
1007 #  endif
1008 #endif /* AO_HAVE_compare_and_swap_full */
1009
1010 #if !defined(AO_HAVE_compare_and_swap) && \
1011     defined(AO_HAVE_compare_and_swap_release)
1012 #  define AO_compare_and_swap(addr, old, new_val) \
1013         AO_compare_and_swap_release(addr, old, new_val)
1014 #  define AO_HAVE_compare_and_swap
1015 #endif
1016 #if !defined(AO_HAVE_compare_and_swap) && \
1017     defined(AO_HAVE_compare_and_swap_acquire)
1018 #  define AO_compare_and_swap(addr, old, new_val) \
1019         AO_compare_and_swap_acquire(addr, old, new_val)
1020 #  define AO_HAVE_compare_and_swap
1021 #endif
1022 #if !defined(AO_HAVE_compare_and_swap) && \
1023     defined(AO_HAVE_compare_and_swap_write)
1024 #  define AO_compare_and_swap(addr, old, new_val) \
1025         AO_compare_and_swap_write(addr, old, new_val)
1026 #  define AO_HAVE_compare_and_swap
1027 #endif
1028 #if !defined(AO_HAVE_compare_and_swap) && \
1029     defined(AO_HAVE_compare_and_swap_read)
1030 #  define AO_compare_and_swap(addr, old, new_val) \
1031         AO_compare_and_swap_read(addr, old, new_val)
1032 #  define AO_HAVE_compare_and_swap
1033 #endif
1034
1035 #if defined(AO_HAVE_compare_and_swap_acquire) &&\
1036     defined(AO_HAVE_nop_full) && \
1037     !defined(AO_HAVE_compare_and_swap_full)
1038 #  define AO_compare_and_swap_full(addr, old, new_val) \
1039         (AO_nop_full(), AO_compare_and_swap_acquire(addr, old, new_val))
1040 #  define AO_HAVE_compare_and_swap_full
1041 #endif
1042
1043 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
1044     defined(AO_HAVE_compare_and_swap_write)
1045 #  define AO_compare_and_swap_release_write(addr, old, new_val) \
1046         AO_compare_and_swap_write(addr, old, new_val)
1047 #  define AO_HAVE_compare_and_swap_release_write
1048 #endif
1049 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
1050     defined(AO_HAVE_compare_and_swap_release)
1051 #  define AO_compare_and_swap_release_write(addr, old, new_val) \
1052         AO_compare_and_swap_release(addr, old, new_val)
1053 #  define AO_HAVE_compare_and_swap_release_write
1054 #endif
1055 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
1056     defined(AO_HAVE_compare_and_swap_read)
1057 #  define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1058         AO_compare_and_swap_read(addr, old, new_val)
1059 #  define AO_HAVE_compare_and_swap_acquire_read
1060 #endif
1061 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
1062     defined(AO_HAVE_compare_and_swap_acquire)
1063 #  define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1064         AO_compare_and_swap_acquire(addr, old, new_val)
1065 #  define AO_HAVE_compare_and_swap_acquire_read
1066 #endif
1067
1068 #ifdef AO_NO_DD_ORDERING
1069 #  if defined(AO_HAVE_compare_and_swap_acquire_read)
1070 #    define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1071         AO_compare_and_swap_acquire_read(addr, old, new_val)
1072 #    define AO_HAVE_compare_and_swap_dd_acquire_read
1073 #  endif
1074 #else
1075 #  if defined(AO_HAVE_compare_and_swap)
1076 #    define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1077         AO_compare_and_swap(addr, old, new_val)
1078 #    define AO_HAVE_compare_and_swap_dd_acquire_read
1079 #  endif
1080 #endif
1081
1082 #include "generalize-small.h"
1083
1084 /* Compare_double_and_swap_double */
1085 #if defined(AO_HAVE_compare_double_and_swap_double) && defined(AO_HAVE_nop_full)\
1086     && !defined(AO_HAVE_compare_double_and_swap_double_acquire)
1087    AO_INLINE int
1088    AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
1089                                              AO_t o1, AO_t o2,
1090                                              AO_t n1, AO_t n2)
1091    {
1092      int result = AO_compare_double_and_swap_double(addr, o1, o2, n1, n2);
1093      AO_nop_full();
1094      return result;
1095    }
1096 #  define AO_HAVE_compare_double_and_swap_double_acquire
1097 #endif
1098 #if defined(AO_HAVE_compare_double_and_swap_double) \
1099     && defined(AO_HAVE_nop_full)\
1100     && !defined(AO_HAVE_compare_double_and_swap_double_release)
1101 #  define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
1102         (AO_nop_full(), AO_compare_double_and_swap_double(addr, o1, o2, n1, n2))
1103 #  define AO_HAVE_compare_double_and_swap_double_release
1104 #endif
1105 #if defined(AO_HAVE_compare_double_and_swap_double_full)
1106 #  if !defined(AO_HAVE_compare_double_and_swap_double_release)
1107 #    define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
1108          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1109 #    define AO_HAVE_compare_double_and_swap_double_release
1110 #  endif
1111 #  if !defined(AO_HAVE_compare_double_and_swap_double_acquire)
1112 #    define AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2) \
1113          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1114 #    define AO_HAVE_compare_double_and_swap_double_acquire
1115 #  endif
1116 #  if !defined(AO_HAVE_compare_double_and_swap_double_write)
1117 #    define AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2) \
1118          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1119 #    define AO_HAVE_compare_double_and_swap_double_write
1120 #  endif
1121 #  if !defined(AO_HAVE_compare_double_and_swap_double_read)
1122 #    define AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2) \
1123          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1124 #    define AO_HAVE_compare_double_and_swap_double_read
1125 #  endif
1126 #endif /* AO_HAVE_compare_double_and_swap_double_full */
1127
1128 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1129     defined(AO_HAVE_compare_double_and_swap_double_release)
1130 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1131         AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
1132 #  define AO_HAVE_compare_double_and_swap_double
1133 #endif
1134 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1135     defined(AO_HAVE_compare_double_and_swap_double_acquire)
1136 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1137         AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
1138 #  define AO_HAVE_compare_double_and_swap_double
1139 #endif
1140 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1141     defined(AO_HAVE_compare_double_and_swap_double_write)
1142 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1143         AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
1144 #  define AO_HAVE_compare_double_and_swap_double
1145 #endif
1146 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1147     defined(AO_HAVE_compare_double_and_swap_double_read)
1148 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1149         AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
1150 #  define AO_HAVE_compare_double_and_swap_double
1151 #endif
1152
1153 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) &&\
1154     defined(AO_HAVE_nop_full) && \
1155     !defined(AO_HAVE_compare_double_and_swap_double_full)
1156 #  define AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2) \
1157         (AO_nop_full(), AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2))
1158 #  define AO_HAVE_compare_double_and_swap_double_full
1159 #endif
1160
1161 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
1162     defined(AO_HAVE_compare_double_and_swap_double_write)
1163 #  define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
1164         AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
1165 #  define AO_HAVE_compare_double_and_swap_double_release_write
1166 #endif
1167 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
1168     defined(AO_HAVE_compare_double_and_swap_double_release)
1169 #  define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
1170         AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
1171 #  define AO_HAVE_compare_double_and_swap_double_release_write
1172 #endif
1173 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
1174     defined(AO_HAVE_compare_double_and_swap_double_read)
1175 #  define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
1176         AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
1177 #  define AO_HAVE_compare_double_and_swap_double_acquire_read
1178 #endif
1179 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
1180     defined(AO_HAVE_compare_double_and_swap_double_acquire)
1181 #  define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
1182         AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
1183 #  define AO_HAVE_compare_double_and_swap_double_acquire_read
1184 #endif
1185
1186 #ifdef AO_NO_DD_ORDERING
1187 #  if defined(AO_HAVE_compare_double_and_swap_double_acquire_read)
1188 #    define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
1189         AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2)
1190 #    define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
1191 #  endif
1192 #else
1193 #  if defined(AO_HAVE_compare_double_and_swap_double)
1194 #    define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
1195         AO_compare_double_and_swap_double(addr, o1, o2, n1, n2)
1196 #    define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
1197 #  endif
1198 #endif
1199
1200 /* Compare_and_swap_double */
1201 #if defined(AO_HAVE_compare_and_swap_double) && defined(AO_HAVE_nop_full)\
1202     && !defined(AO_HAVE_compare_and_swap_double_acquire)
1203    AO_INLINE int
1204    AO_compare_and_swap_double_acquire(volatile AO_double_t *addr,
1205                                              AO_t o1,
1206                                              AO_t n1, AO_t n2)
1207    {
1208      int result = AO_compare_and_swap_double(addr, o1, n1, n2);
1209      AO_nop_full();
1210      return result;
1211    }
1212 #  define AO_HAVE_compare_and_swap_double_acquire
1213 #endif
1214 #if defined(AO_HAVE_compare_and_swap_double) \
1215     && defined(AO_HAVE_nop_full)\
1216     && !defined(AO_HAVE_compare_and_swap_double_release)
1217 #  define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
1218         (AO_nop_full(), AO_compare_and_swap_double(addr, o1, n1, n2))
1219 #  define AO_HAVE_compare_and_swap_double_release
1220 #endif
1221 #if defined(AO_HAVE_compare_and_swap_double_full)
1222 #  if !defined(AO_HAVE_compare_and_swap_double_release)
1223 #    define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
1224          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1225 #    define AO_HAVE_compare_and_swap_double_release
1226 #  endif
1227 #  if !defined(AO_HAVE_compare_and_swap_double_acquire)
1228 #    define AO_compare_and_swap_double_acquire(addr, o1, n1, n2) \
1229          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1230 #    define AO_HAVE_compare_and_swap_double_acquire
1231 #  endif
1232 #  if !defined(AO_HAVE_compare_and_swap_double_write)
1233 #    define AO_compare_and_swap_double_write(addr, o1, n1, n2) \
1234          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1235 #    define AO_HAVE_compare_and_swap_double_write
1236 #  endif
1237 #  if !defined(AO_HAVE_compare_and_swap_double_read)
1238 #    define AO_compare_and_swap_double_read(addr, o1, n1, n2) \
1239          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1240 #    define AO_HAVE_compare_and_swap_double_read
1241 #  endif
1242 #endif /* AO_HAVE_compare_and_swap_double_full */
1243
1244 #if !defined(AO_HAVE_compare_and_swap_double) && \
1245     defined(AO_HAVE_compare_and_swap_double_release)
1246 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1247         AO_compare_and_swap_double_release(addr, o1, n1, n2)
1248 #  define AO_HAVE_compare_and_swap_double
1249 #endif
1250 #if !defined(AO_HAVE_compare_and_swap_double) && \
1251     defined(AO_HAVE_compare_and_swap_double_acquire)
1252 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1253         AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
1254 #  define AO_HAVE_compare_and_swap_double
1255 #endif
1256 #if !defined(AO_HAVE_compare_and_swap_double) && \
1257     defined(AO_HAVE_compare_and_swap_double_write)
1258 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1259         AO_compare_and_swap_double_write(addr, o1, n1, n2)
1260 #  define AO_HAVE_compare_and_swap_double
1261 #endif
1262 #if !defined(AO_HAVE_compare_and_swap_double) && \
1263     defined(AO_HAVE_compare_and_swap_double_read)
1264 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1265         AO_compare_and_swap_double_read(addr, o1, n1, n2)
1266 #  define AO_HAVE_compare_and_swap_double
1267 #endif
1268
1269 #if defined(AO_HAVE_compare_and_swap_double_acquire) &&\
1270     defined(AO_HAVE_nop_full) && \
1271     !defined(AO_HAVE_compare_and_swap_double_full)
1272 #  define AO_compare_and_swap_double_full(addr, o1, n1, n2) \
1273         (AO_nop_full(), AO_compare_and_swap_double_acquire(addr, o1, n1, n2))
1274 #  define AO_HAVE_compare_and_swap_double_full
1275 #endif
1276
1277 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
1278     defined(AO_HAVE_compare_and_swap_double_write)
1279 #  define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
1280         AO_compare_and_swap_double_write(addr, o1, n1, n2)
1281 #  define AO_HAVE_compare_and_swap_double_release_write
1282 #endif
1283 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
1284     defined(AO_HAVE_compare_and_swap_double_release)
1285 #  define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
1286         AO_compare_and_swap_double_release(addr, o1, n1, n2)
1287 #  define AO_HAVE_compare_and_swap_double_release_write
1288 #endif
1289 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
1290     defined(AO_HAVE_compare_and_swap_double_read)
1291 #  define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
1292         AO_compare_and_swap_double_read(addr, o1, n1, n2)
1293 #  define AO_HAVE_compare_and_swap_double_acquire_read
1294 #endif
1295 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
1296     defined(AO_HAVE_compare_and_swap_double_acquire)
1297 #  define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
1298         AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
1299 #  define AO_HAVE_compare_and_swap_double_acquire_read
1300 #endif
1301
1302 #ifdef AO_NO_DD_ORDERING
1303 #  if defined(AO_HAVE_compare_and_swap_double_acquire_read)
1304 #    define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
1305         AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2)
1306 #    define AO_HAVE_compare_and_swap_double_dd_acquire_read
1307 #  endif
1308 #else
1309 #  if defined(AO_HAVE_compare_and_swap_double)
1310 #    define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
1311         AO_compare_and_swap_double(addr, o1, n1, n2)
1312 #    define AO_HAVE_compare_and_swap_double_dd_acquire_read
1313 #  endif
1314 #endif
1315
1316 /* NEC LE-IT: Convenience functions for AO_double compare and swap which */
1317 /* types and reads easier in code                                        */
1318 #if defined(AO_HAVE_compare_double_and_swap_double_release) && \
1319     !defined(AO_HAVE_double_compare_and_swap_release)
1320 AO_INLINE int
1321 AO_double_compare_and_swap_release(volatile AO_double_t *addr,
1322                                    AO_double_t old_val, AO_double_t new_val)
1323 {
1324         return AO_compare_double_and_swap_double_release(addr,
1325                                                          old_val.AO_val1, old_val.AO_val2,
1326                                                          new_val.AO_val1, new_val.AO_val2);
1327 }
1328 #define AO_HAVE_double_compare_and_swap_release
1329 #endif
1330
1331 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) && \
1332     !defined(AO_HAVE_double_compare_and_swap_acquire)
1333 AO_INLINE int
1334 AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
1335                                    AO_double_t old_val, AO_double_t new_val)
1336 {
1337         return AO_compare_double_and_swap_double_acquire(addr,
1338                                                          old_val.AO_val1, old_val.AO_val2,
1339                                                          new_val.AO_val1, new_val.AO_val2);
1340 }
1341 #define AO_HAVE_double_compare_and_swap_acquire
1342 #endif
1343
1344 #if defined(AO_HAVE_compare_double_and_swap_double_full) && \
1345     !defined(AO_HAVE_double_compare_and_swap_full)
1346 AO_INLINE int
1347 AO_double_compare_and_swap_full(volatile AO_double_t *addr,
1348                                          AO_double_t old_val, AO_double_t new_val)
1349 {
1350         return AO_compare_double_and_swap_double_full(addr,
1351                                                       old_val.AO_val1, old_val.AO_val2,
1352                                                       new_val.AO_val1, new_val.AO_val2);
1353 }
1354 #define AO_HAVE_double_compare_and_swap_full
1355 #endif