src/mm/boehm-gc/libatomic_ops-1.2/src/atomic_ops/generalize.h

   1 /*
   2  * Copyright (c) 2003-2004 Hewlett-Packard Development Company, L.P.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a copy
   5  * of this software and associated documentation files (the "Software"), to deal
   6  * in the Software without restriction, including without limitation the rights
   7  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8  * copies of the Software, and to permit persons to whom the Software is
   9  * furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20  * SOFTWARE.
  21  */
  22
  23 /*
  24  * Generalize atomic operations for atomic_ops.h.
  25  * Should not be included directly.
  26  *
  27  * We make no attempt to define useless operations, such as
  28  * AO_nop_acquire
  29  * AO_nop_release
  30  *
  31  * We have also so far neglected to define some others, which
  32  * do not appear likely to be useful, e.g. stores with acquire
  33  * or read barriers.
  34  *
  35  * This file is sometimes included twice by atomic_ops.h.
  36  * All definitions include explicit checks that we are not replacing
  37  * an earlier definition.  In general, more desirable expansions
  38  * appear earlier so that we are more likely to use them.
  39  *
  40  * We only make safe generalizations, except that by default we define
  41  * the ...dd_acquire_read operations to be equivalent to those without
  42  * a barrier.  On platforms for which this is unsafe, the platform-specific
  43  * file must define AO_NO_DD_ORDERING.
  44  */
  45
  46 #ifndef ATOMIC_OPS_H
  47 # error Atomic_ops_generalize.h should not be included directly.
  48 #endif
  49
  50 #if AO_CHAR_TS_T
  51 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
  52          AO_char_compare_and_swap_full(a,o,n)
  53 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
  54          AO_char_compare_and_swap_acquire(a,o,n)
  55 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
  56          AO_char_compare_and_swap_release(a,o,n)
  57 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
  58          AO_char_compare_and_swap(a,o,n)
  59 #endif
  60
  61 #if AO_AO_TS_T
  62 # define AO_TS_COMPARE_AND_SWAP_FULL(a,o,n) \
  63          AO_compare_and_swap_full(a,o,n)
  64 # define AO_TS_COMPARE_AND_SWAP_ACQUIRE(a,o,n) \
  65          AO_compare_and_swap_acquire(a,o,n)
  66 # define AO_TS_COMPARE_AND_SWAP_RELEASE(a,o,n) \
  67          AO_compare_and_swap_release(a,o,n)
  68 # define AO_TS_COMPARE_AND_SWAP(a,o,n) \
  69          AO_compare_and_swap(a,o,n)
  70 #endif
  71
  72 /* Generate test_and_set_full, if necessary and possible.       */
  73 #if !defined(AO_HAVE_test_and_set) && \
  74     !defined(AO_HAVE_test_and_set_release) && \
  75     !defined(AO_HAVE_test_and_set_acquire) && \
  76     !defined(AO_HAVE_test_and_set_read) && \
  77     !defined(AO_HAVE_test_and_set_full)
  78 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_full) || \
  79       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_full)
  80      AO_INLINE AO_TS_VAL_t
  81      AO_test_and_set_full(volatile AO_TS_t *addr)
  82      {
  83        if (AO_TS_COMPARE_AND_SWAP_FULL(addr, AO_TS_CLEAR, AO_TS_SET))
  84          return AO_TS_CLEAR;
  85        else
  86          return AO_TS_SET;
  87      }
  88 #    define AO_HAVE_test_and_set_full
  89 #  endif /* AO_HAVE_compare_and_swap_full */
  90
  91 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_acquire) || \
  92       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_acquire)
  93      AO_INLINE AO_TS_VAL_t
  94      AO_test_and_set_acquire(volatile AO_TS_t *addr)
  95      {
  96        if (AO_TS_COMPARE_AND_SWAP_ACQUIRE(addr, AO_TS_CLEAR, AO_TS_SET))
  97          return AO_TS_CLEAR;
  98        else
  99          return AO_TS_SET;
 100      }
 101 #    define AO_HAVE_test_and_set_acquire
 102 #  endif /* AO_HAVE_compare_and_swap_acquire */
 103
 104 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap_release) || \
 105       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap_release)
 106      AO_INLINE AO_TS_VAL_t
 107      AO_test_and_set_release(volatile AO_TS_t *addr)
 108      {
 109        if (AO_TS_COMPARE_AND_SWAP_RELEASE(addr, AO_TS_CLEAR, AO_TS_SET))
 110          return AO_TS_CLEAR;
 111        else
 112          return AO_TS_SET;
 113      }
 114 #    define AO_HAVE_test_and_set_release
 115 #  endif /* AO_HAVE_compare_and_swap_release */
 116
 117 #  if AO_AO_TS_T && defined(AO_HAVE_compare_and_swap) || \
 118       AO_CHAR_TS_T && defined(AO_HAVE_char_compare_and_swap)
 119      AO_INLINE AO_TS_VAL_t
 120      AO_test_and_set(volatile AO_TS_t *addr)
 121      {
 122        if (AO_TS_COMPARE_AND_SWAP(addr, AO_TS_CLEAR, AO_TS_SET))
 123          return AO_TS_CLEAR;
 124        else
 125          return AO_TS_SET;
 126      }
 127 #    define AO_HAVE_test_and_set
 128 #  endif /* AO_HAVE_compare_and_swap */
 129
 130 #  if defined(AO_HAVE_test_and_set) && defined(AO_HAVE_nop_full) \
 131       && !defined(AO_HAVE_test_and_set_acquire)
 132      AO_INLINE AO_TS_VAL_t
 133      AO_test_and_set_acquire(volatile AO_TS_t *addr)
 134      {
 135        AO_TS_VAL_t result = AO_test_and_set(addr);
 136        AO_nop_full();
 137        return result;
 138      }
 139 #    define AO_HAVE_test_and_set_acquire
 140 #  endif
 141
 142 #endif /* No prior test and set */
 143
 144 /* Nop */
 145 #if !defined(AO_HAVE_nop)
 146    AO_INLINE void AO_nop(void) {}
 147 #  define AO_HAVE_nop
 148 #endif
 149
 150 #if defined(AO_HAVE_test_and_set_full) && !defined(AO_HAVE_nop_full)
 151    AO_INLINE void
 152    AO_nop_full()
 153    {
 154      AO_TS_t dummy = AO_TS_INITIALIZER;
 155      AO_test_and_set_full(&dummy);
 156    }
 157 #  define AO_HAVE_nop_full
 158 #endif
 159
 160 #if defined(AO_HAVE_nop_acquire)
 161 #  error AO_nop_acquire is useless: dont define.
 162 #endif
 163 #if defined(AO_HAVE_nop_release)
 164 #  error AO_nop_release is useless: dont define.
 165 #endif
 166
 167 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_read)
 168 #  define AO_nop_read() AO_nop_full()
 169 #  define AO_HAVE_nop_read
 170 #endif
 171
 172 #if defined(AO_HAVE_nop_full) && !defined(AO_HAVE_nop_write)
 173 #  define AO_nop_write() AO_nop_full()
 174 #  define AO_HAVE_nop_write
 175 #endif
 176
 177 /* Load */
 178 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_acquire)
 179 #  define AO_load_acquire(addr) AO_load_full(addr)
 180 #  define AO_HAVE_load_acquire
 181 #endif
 182
 183 #if defined(AO_HAVE_load_acquire) && !defined(AO_HAVE_load)
 184 #  define AO_load(addr) AO_load_acquire(addr)
 185 #  define AO_HAVE_load
 186 #endif
 187
 188 #if defined(AO_HAVE_load_full) && !defined(AO_HAVE_load_read)
 189 #  define AO_load_read(addr) AO_load_full(addr)
 190 #  define AO_HAVE_load_read
 191 #endif
 192
 193 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_acquire)
 194 #  define AO_load_acquire_read(addr) AO_load_acquire(addr)
 195 #  define AO_HAVE_load_acquire_read
 196 #endif
 197
 198 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_full) && \
 199     !defined(AO_HAVE_load_acquire)
 200    AO_INLINE AO_t
 201    AO_load_acquire(volatile AO_t *addr)
 202    {
 203      AO_t result = AO_load(addr);
 204      /* Acquire barrier would be useless, since the load could be delayed  */
 205      /* beyond it.                                                         */
 206      AO_nop_full();
 207      return result;
 208    }
 209 #  define AO_HAVE_load_acquire
 210 #endif
 211
 212 #if defined(AO_HAVE_load) && defined(AO_HAVE_nop_read) && \
 213     !defined(AO_HAVE_load_read)
 214    AO_INLINE AO_t
 215    AO_load_read(volatile AO_t *addr)
 216    {
 217      AO_t result = AO_load(addr);
 218      /* Acquire barrier would be useless, since the load could be delayed  */
 219      /* beyond it.                                                         */
 220      AO_nop_read();
 221      return result;
 222    }
 223 #  define AO_HAVE_load_read
 224 #endif
 225
 226 #if defined(AO_HAVE_load_acquire) && defined(AO_HAVE_nop_full) && \
 227     !defined(AO_HAVE_load_full)
 228 #  define AO_load_full(addr) (AO_nop_full(), AO_load_acquire(addr))
 229 #  define AO_HAVE_load_full
 230 #endif
 231
 232 #if !defined(AO_HAVE_load_acquire_read) && defined(AO_HAVE_load_read)
 233 #  define AO_load_acquire_read(addr) AO_load_read(addr)
 234 #  define AO_HAVE_load_acquire_read
 235 #endif
 236
 237 #if defined(AO_HAVE_load_acquire_read) && !defined(AO_HAVE_load)
 238 #  define AO_load(addr) AO_load_acquire_read(addr)
 239 #  define AO_HAVE_load
 240 #endif
 241
 242 #ifdef AO_NO_DD_ORDERING
 243 #  if defined(AO_HAVE_load_acquire_read)
 244 #    define AO_load_dd_acquire_read(addr) AO_load_acquire_read(addr)
 245 #    define AO_HAVE_load_dd_acquire_read
 246 #  endif
 247 #else
 248 #  if defined(AO_HAVE_load)
 249 #    define AO_load_dd_acquire_read(addr) AO_load(addr)
 250 #    define AO_HAVE_load_dd_acquire_read
 251 #  endif
 252 #endif
 253
 254
 255 /* Store */
 256
 257 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_release)
 258 #  define AO_store_release(addr,val) AO_store_full(addr,val)
 259 #  define AO_HAVE_store_release
 260 #endif
 261
 262 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store)
 263 #  define AO_store(addr, val) AO_store_release(addr,val)
 264 #  define AO_HAVE_store
 265 #endif
 266
 267 #if defined(AO_HAVE_store_full) && !defined(AO_HAVE_store_write)
 268 #  define AO_store_write(addr,val) AO_store_full(addr,val)
 269 #  define AO_HAVE_store_write
 270 #endif
 271
 272 #if defined(AO_HAVE_store_release) && !defined(AO_HAVE_store_release_write)
 273 #  define AO_store_release_write(addr, val) AO_store_release(addr,val)
 274 #  define AO_HAVE_store_release_write
 275 #endif
 276
 277 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store)
 278 #  define AO_store(addr, val) AO_store_write(addr,val)
 279 #  define AO_HAVE_store
 280 #endif
 281
 282 #if defined(AO_HAVE_store) && defined(AO_HAVE_nop_full) && \
 283     !defined(AO_HAVE_store_release)
 284 #  define AO_store_release(addr,val) (AO_nop_full(), AO_store(addr,val))
 285 #  define AO_HAVE_store_release
 286 #endif
 287
 288 #if defined(AO_HAVE_nop_write) && defined(AO_HAVE_store) && \
 289      !defined(AO_HAVE_store_write)
 290 #  define AO_store_write(addr, val) (AO_nop_write(), AO_store(addr,val))
 291 #  define AO_HAVE_store_write
 292 #endif
 293
 294 #if defined(AO_HAVE_store_write) && !defined(AO_HAVE_store_release_write)
 295 #  define AO_store_release_write(addr, val) AO_store_write(addr,val)
 296 #  define AO_HAVE_store_release_write
 297 #endif
 298
 299 #if defined(AO_HAVE_store_release) && defined(AO_HAVE_nop_full) && \
 300     !defined(AO_HAVE_store_full)
 301 #  define AO_store_full(addr, val) (AO_store_release(addr, val), AO_nop_full())
 302 #  define AO_HAVE_store_full
 303 #endif
 304
 305
 306 /* Fetch_and_add */
 307 /* We first try to implement fetch_and_add variants in terms    */
 308 /* of the corresponding compare_and_swap variants to minimize   */
 309 /* adding barriers.                                             */
 310 #if defined(AO_HAVE_compare_and_swap_full) && \
 311     !defined(AO_HAVE_fetch_and_add_full)
 312    AO_INLINE AO_t
 313    AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr)
 314    {
 315      AO_t old;
 316      do
 317        {
 318          old = *addr;
 319        }
 320      while (!AO_compare_and_swap_full(addr, old, old+incr));
 321      return old;
 322    }
 323 #  define AO_HAVE_fetch_and_add_full
 324 #endif
 325
 326 #if defined(AO_HAVE_compare_and_swap_acquire) && \
 327     !defined(AO_HAVE_fetch_and_add_acquire)
 328    AO_INLINE AO_t
 329    AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr)
 330    {
 331      AO_t old;
 332      do
 333        {
 334          old = *addr;
 335        }
 336      while (!AO_compare_and_swap_acquire(addr, old, old+incr));
 337      return old;
 338    }
 339 #  define AO_HAVE_fetch_and_add_acquire
 340 #endif
 341
 342 #if defined(AO_HAVE_compare_and_swap_release) && \
 343     !defined(AO_HAVE_fetch_and_add_release)
 344    AO_INLINE AO_t
 345    AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr)
 346    {
 347      AO_t old;
 348      do
 349        {
 350          old = *addr;
 351        }
 352      while (!AO_compare_and_swap_release(addr, old, old+incr));
 353      return old;
 354    }
 355 #  define AO_HAVE_fetch_and_add_release
 356 #endif
 357
 358 #if defined(AO_HAVE_compare_and_swap) && \
 359     !defined(AO_HAVE_fetch_and_add)
 360    AO_INLINE AO_t
 361    AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
 362    {
 363      AO_t old;
 364      do
 365        {
 366          old = *addr;
 367        }
 368      while (!AO_compare_and_swap(addr, old, old+incr));
 369      return old;
 370    }
 371 #  define AO_HAVE_fetch_and_add
 372 #endif
 373
 374 #if defined(AO_HAVE_fetch_and_add_full)
 375 #  if !defined(AO_HAVE_fetch_and_add_release)
 376 #    define AO_fetch_and_add_release(addr, val) \
 377          AO_fetch_and_add_full(addr, val)
 378 #    define AO_HAVE_fetch_and_add_release
 379 #  endif
 380 #  if !defined(AO_HAVE_fetch_and_add_acquire)
 381 #    define AO_fetch_and_add_acquire(addr, val) \
 382          AO_fetch_and_add_full(addr, val)
 383 #    define AO_HAVE_fetch_and_add_acquire
 384 #  endif
 385 #  if !defined(AO_HAVE_fetch_and_add_write)
 386 #    define AO_fetch_and_add_write(addr, val) \
 387          AO_fetch_and_add_full(addr, val)
 388 #    define AO_HAVE_fetch_and_add_write
 389 #  endif
 390 #  if !defined(AO_HAVE_fetch_and_add_read)
 391 #    define AO_fetch_and_add_read(addr, val) \
 392          AO_fetch_and_add_full(addr, val)
 393 #    define AO_HAVE_fetch_and_add_read
 394 #  endif
 395 #endif /* AO_HAVE_fetch_and_add_full */
 396
 397 #if !defined(AO_HAVE_fetch_and_add) && \
 398     defined(AO_HAVE_fetch_and_add_release)
 399 #  define AO_fetch_and_add(addr, val) \
 400         AO_fetch_and_add_release(addr, val)
 401 #  define AO_HAVE_fetch_and_add
 402 #endif
 403 #if !defined(AO_HAVE_fetch_and_add) && \
 404     defined(AO_HAVE_fetch_and_add_acquire)
 405 #  define AO_fetch_and_add(addr, val) \
 406         AO_fetch_and_add_acquire(addr, val)
 407 #  define AO_HAVE_fetch_and_add
 408 #endif
 409 #if !defined(AO_HAVE_fetch_and_add) && \
 410     defined(AO_HAVE_fetch_and_add_write)
 411 #  define AO_fetch_and_add(addr, val) \
 412         AO_fetch_and_add_write(addr, val)
 413 #  define AO_HAVE_fetch_and_add
 414 #endif
 415 #if !defined(AO_HAVE_fetch_and_add) && \
 416     defined(AO_HAVE_fetch_and_add_read)
 417 #  define AO_fetch_and_add(addr, val) \
 418         AO_fetch_and_add_read(addr, val)
 419 #  define AO_HAVE_fetch_and_add
 420 #endif
 421
 422 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
 423     defined(AO_HAVE_nop_full) && \
 424     !defined(AO_HAVE_fetch_and_add_full)
 425 #  define AO_fetch_and_add_full(addr, val) \
 426         (AO_nop_full(), AO_fetch_and_add_acquire(addr, val))
 427 #  define AO_HAVE_fetch_and_add_full
 428 #endif
 429
 430 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
 431     defined(AO_HAVE_fetch_and_add_write)
 432 #  define AO_fetch_and_add_release_write(addr, val) \
 433         AO_fetch_and_add_write(addr, val)
 434 #  define AO_HAVE_fetch_and_add_release_write
 435 #endif
 436 #if !defined(AO_HAVE_fetch_and_add_release_write) && \
 437     defined(AO_HAVE_fetch_and_add_release)
 438 #  define AO_fetch_and_add_release_write(addr, val) \
 439         AO_fetch_and_add_release(addr, val)
 440 #  define AO_HAVE_fetch_and_add_release_write
 441 #endif
 442 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
 443     defined(AO_HAVE_fetch_and_add_read)
 444 #  define AO_fetch_and_add_acquire_read(addr, val) \
 445         AO_fetch_and_add_read(addr, val)
 446 #  define AO_HAVE_fetch_and_add_acquire_read
 447 #endif
 448 #if !defined(AO_HAVE_fetch_and_add_acquire_read) && \
 449     defined(AO_HAVE_fetch_and_add_acquire)
 450 #  define AO_fetch_and_add_acquire_read(addr, val) \
 451         AO_fetch_and_add_acquire(addr, val)
 452 #  define AO_HAVE_fetch_and_add_acquire_read
 453 #endif
 454
 455 #ifdef AO_NO_DD_ORDERING
 456 #  if defined(AO_HAVE_fetch_and_add_acquire_read)
 457 #    define AO_fetch_and_add_dd_acquire_read(addr, val) \
 458         AO_fetch_and_add_acquire_read(addr, val)
 459 #    define AO_HAVE_fetch_and_add_dd_acquire_read
 460 #  endif
 461 #else
 462 #  if defined(AO_HAVE_fetch_and_add)
 463 #    define AO_fetch_and_add_dd_acquire_read(addr, val) \
 464         AO_fetch_and_add(addr, val)
 465 #    define AO_HAVE_fetch_and_add_dd_acquire_read
 466 #  endif
 467 #endif
 468
 469 /* Fetch_and_add1 */
 470
 471 #if defined(AO_HAVE_fetch_and_add_full) &&\
 472     !defined(AO_HAVE_fetch_and_add1_full)
 473 #  define AO_fetch_and_add1_full(addr) AO_fetch_and_add_full(addr,1)
 474 #  define AO_HAVE_fetch_and_add1_full
 475 #endif
 476 #if defined(AO_HAVE_fetch_and_add_release) &&\
 477     !defined(AO_HAVE_fetch_and_add1_release)
 478 #  define AO_fetch_and_add1_release(addr) AO_fetch_and_add_release(addr,1)
 479 #  define AO_HAVE_fetch_and_add1_release
 480 #endif
 481 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
 482     !defined(AO_HAVE_fetch_and_add1_acquire)
 483 #  define AO_fetch_and_add1_acquire(addr) AO_fetch_and_add_acquire(addr,1)
 484 #  define AO_HAVE_fetch_and_add1_acquire
 485 #endif
 486 #if defined(AO_HAVE_fetch_and_add_write) &&\
 487     !defined(AO_HAVE_fetch_and_add1_write)
 488 #  define AO_fetch_and_add1_write(addr) AO_fetch_and_add_write(addr,1)
 489 #  define AO_HAVE_fetch_and_add1_write
 490 #endif
 491 #if defined(AO_HAVE_fetch_and_add_read) &&\
 492     !defined(AO_HAVE_fetch_and_add1_read)
 493 #  define AO_fetch_and_add1_read(addr) AO_fetch_and_add_read(addr,1)
 494 #  define AO_HAVE_fetch_and_add1_read
 495 #endif
 496 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
 497     !defined(AO_HAVE_fetch_and_add1_release_write)
 498 #  define AO_fetch_and_add1_release_write(addr) \
 499         AO_fetch_and_add_release_write(addr,1)
 500 #  define AO_HAVE_fetch_and_add1_release_write
 501 #endif
 502 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
 503     !defined(AO_HAVE_fetch_and_add1_acquire_read)
 504 #  define AO_fetch_and_add1_acquire_read(addr) \
 505         AO_fetch_and_add_acquire_read(addr,1)
 506 #  define AO_HAVE_fetch_and_add1_acquire_read
 507 #endif
 508 #if defined(AO_HAVE_fetch_and_add) &&\
 509     !defined(AO_HAVE_fetch_and_add1)
 510 #  define AO_fetch_and_add1(addr) \
 511         AO_fetch_and_add(addr,1)
 512 #  define AO_HAVE_fetch_and_add1
 513 #endif
 514
 515 #if defined(AO_HAVE_fetch_and_add1_full)
 516 #  if !defined(AO_HAVE_fetch_and_add1_release)
 517 #    define AO_fetch_and_add1_release(addr) \
 518          AO_fetch_and_add1_full(addr)
 519 #    define AO_HAVE_fetch_and_add1_release
 520 #  endif
 521 #  if !defined(AO_HAVE_fetch_and_add1_acquire)
 522 #    define AO_fetch_and_add1_acquire(addr) \
 523          AO_fetch_and_add1_full(addr)
 524 #    define AO_HAVE_fetch_and_add1_acquire
 525 #  endif
 526 #  if !defined(AO_HAVE_fetch_and_add1_write)
 527 #    define AO_fetch_and_add1_write(addr) \
 528          AO_fetch_and_add1_full(addr)
 529 #    define AO_HAVE_fetch_and_add1_write
 530 #  endif
 531 #  if !defined(AO_HAVE_fetch_and_add1_read)
 532 #    define AO_fetch_and_add1_read(addr) \
 533          AO_fetch_and_add1_full(addr)
 534 #    define AO_HAVE_fetch_and_add1_read
 535 #  endif
 536 #endif /* AO_HAVE_fetch_and_add1_full */
 537
 538 #if !defined(AO_HAVE_fetch_and_add1) && \
 539     defined(AO_HAVE_fetch_and_add1_release)
 540 #  define AO_fetch_and_add1(addr) \
 541         AO_fetch_and_add1_release(addr)
 542 #  define AO_HAVE_fetch_and_add1
 543 #endif
 544 #if !defined(AO_HAVE_fetch_and_add1) && \
 545     defined(AO_HAVE_fetch_and_add1_acquire)
 546 #  define AO_fetch_and_add1(addr) \
 547         AO_fetch_and_add1_acquire(addr)
 548 #  define AO_HAVE_fetch_and_add1
 549 #endif
 550 #if !defined(AO_HAVE_fetch_and_add1) && \
 551     defined(AO_HAVE_fetch_and_add1_write)
 552 #  define AO_fetch_and_add1(addr) \
 553         AO_fetch_and_add1_write(addr)
 554 #  define AO_HAVE_fetch_and_add1
 555 #endif
 556 #if !defined(AO_HAVE_fetch_and_add1) && \
 557     defined(AO_HAVE_fetch_and_add1_read)
 558 #  define AO_fetch_and_add1(addr) \
 559         AO_fetch_and_add1_read(addr)
 560 #  define AO_HAVE_fetch_and_add1
 561 #endif
 562
 563 #if defined(AO_HAVE_fetch_and_add1_acquire) &&\
 564     defined(AO_HAVE_nop_full) && \
 565     !defined(AO_HAVE_fetch_and_add1_full)
 566 #  define AO_fetch_and_add1_full(addr) \
 567         (AO_nop_full(), AO_fetch_and_add1_acquire(addr))
 568 #  define AO_HAVE_fetch_and_add1_full
 569 #endif
 570
 571 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
 572     defined(AO_HAVE_fetch_and_add1_write)
 573 #  define AO_fetch_and_add1_release_write(addr) \
 574         AO_fetch_and_add1_write(addr)
 575 #  define AO_HAVE_fetch_and_add1_release_write
 576 #endif
 577 #if !defined(AO_HAVE_fetch_and_add1_release_write) && \
 578     defined(AO_HAVE_fetch_and_add1_release)
 579 #  define AO_fetch_and_add1_release_write(addr) \
 580         AO_fetch_and_add1_release(addr)
 581 #  define AO_HAVE_fetch_and_add1_release_write
 582 #endif
 583 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
 584     defined(AO_HAVE_fetch_and_add1_read)
 585 #  define AO_fetch_and_add1_acquire_read(addr) \
 586         AO_fetch_and_add1_read(addr)
 587 #  define AO_HAVE_fetch_and_add1_acquire_read
 588 #endif
 589 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) && \
 590     defined(AO_HAVE_fetch_and_add1_acquire)
 591 #  define AO_fetch_and_add1_acquire_read(addr) \
 592         AO_fetch_and_add1_acquire(addr)
 593 #  define AO_HAVE_fetch_and_add1_acquire_read
 594 #endif
 595
 596 #ifdef AO_NO_DD_ORDERING
 597 #  if defined(AO_HAVE_fetch_and_add1_acquire_read)
 598 #    define AO_fetch_and_add1_dd_acquire_read(addr) \
 599         AO_fetch_and_add1_acquire_read(addr)
 600 #    define AO_HAVE_fetch_and_add1_dd_acquire_read
 601 #  endif
 602 #else
 603 #  if defined(AO_HAVE_fetch_and_add1)
 604 #    define AO_fetch_and_add1_dd_acquire_read(addr) AO_fetch_and_add1(addr)
 605 #    define AO_HAVE_fetch_and_add1_dd_acquire_read
 606 #  endif
 607 #endif
 608
 609 /* Fetch_and_sub1 */
 610
 611 #if defined(AO_HAVE_fetch_and_add_full) &&\
 612     !defined(AO_HAVE_fetch_and_sub1_full)
 613 #  define AO_fetch_and_sub1_full(addr) AO_fetch_and_add_full(addr,(AO_t)(-1))
 614 #  define AO_HAVE_fetch_and_sub1_full
 615 #endif
 616 #if defined(AO_HAVE_fetch_and_add_release) &&\
 617     !defined(AO_HAVE_fetch_and_sub1_release)
 618 #  define AO_fetch_and_sub1_release(addr) \
 619         AO_fetch_and_add_release(addr,(AO_t)(-1))
 620 #  define AO_HAVE_fetch_and_sub1_release
 621 #endif
 622 #if defined(AO_HAVE_fetch_and_add_acquire) &&\
 623     !defined(AO_HAVE_fetch_and_sub1_acquire)
 624 #  define AO_fetch_and_sub1_acquire(addr) \
 625         AO_fetch_and_add_acquire(addr,(AO_t)(-1))
 626 #  define AO_HAVE_fetch_and_sub1_acquire
 627 #endif
 628 #if defined(AO_HAVE_fetch_and_add_write) &&\
 629     !defined(AO_HAVE_fetch_and_sub1_write)
 630 #  define AO_fetch_and_sub1_write(addr) \
 631         AO_fetch_and_add_write(addr,(AO_t)(-1))
 632 #  define AO_HAVE_fetch_and_sub1_write
 633 #endif
 634 #if defined(AO_HAVE_fetch_and_add_read) &&\
 635     !defined(AO_HAVE_fetch_and_sub1_read)
 636 #  define AO_fetch_and_sub1_read(addr) \
 637         AO_fetch_and_add_read(addr,(AO_t)(-1))
 638 #  define AO_HAVE_fetch_and_sub1_read
 639 #endif
 640 #if defined(AO_HAVE_fetch_and_add_release_write) &&\
 641     !defined(AO_HAVE_fetch_and_sub1_release_write)
 642 #  define AO_fetch_and_sub1_release_write(addr) \
 643         AO_fetch_and_add_release_write(addr,(AO_t)(-1))
 644 #  define AO_HAVE_fetch_and_sub1_release_write
 645 #endif
 646 #if defined(AO_HAVE_fetch_and_add_acquire_read) &&\
 647     !defined(AO_HAVE_fetch_and_sub1_acquire_read)
 648 #  define AO_fetch_and_sub1_acquire_read(addr) \
 649         AO_fetch_and_add_acquire_read(addr,(AO_t)(-1))
 650 #  define AO_HAVE_fetch_and_sub1_acquire_read
 651 #endif
 652 #if defined(AO_HAVE_fetch_and_add) &&\
 653     !defined(AO_HAVE_fetch_and_sub1)
 654 #  define AO_fetch_and_sub1(addr) \
 655         AO_fetch_and_add(addr,(AO_t)(-1))
 656 #  define AO_HAVE_fetch_and_sub1
 657 #endif
 658
 659 #if defined(AO_HAVE_fetch_and_sub1_full)
 660 #  if !defined(AO_HAVE_fetch_and_sub1_release)
 661 #    define AO_fetch_and_sub1_release(addr) \
 662          AO_fetch_and_sub1_full(addr)
 663 #    define AO_HAVE_fetch_and_sub1_release
 664 #  endif
 665 #  if !defined(AO_HAVE_fetch_and_sub1_acquire)
 666 #    define AO_fetch_and_sub1_acquire(addr) \
 667          AO_fetch_and_sub1_full(addr)
 668 #    define AO_HAVE_fetch_and_sub1_acquire
 669 #  endif
 670 #  if !defined(AO_HAVE_fetch_and_sub1_write)
 671 #    define AO_fetch_and_sub1_write(addr) \
 672          AO_fetch_and_sub1_full(addr)
 673 #    define AO_HAVE_fetch_and_sub1_write
 674 #  endif
 675 #  if !defined(AO_HAVE_fetch_and_sub1_read)
 676 #    define AO_fetch_and_sub1_read(addr) \
 677          AO_fetch_and_sub1_full(addr)
 678 #    define AO_HAVE_fetch_and_sub1_read
 679 #  endif
 680 #endif /* AO_HAVE_fetch_and_sub1_full */
 681
 682 #if !defined(AO_HAVE_fetch_and_sub1) && \
 683     defined(AO_HAVE_fetch_and_sub1_release)
 684 #  define AO_fetch_and_sub1(addr) \
 685         AO_fetch_and_sub1_release(addr)
 686 #  define AO_HAVE_fetch_and_sub1
 687 #endif
 688 #if !defined(AO_HAVE_fetch_and_sub1) && \
 689     defined(AO_HAVE_fetch_and_sub1_acquire)
 690 #  define AO_fetch_and_sub1(addr) \
 691         AO_fetch_and_sub1_acquire(addr)
 692 #  define AO_HAVE_fetch_and_sub1
 693 #endif
 694 #if !defined(AO_HAVE_fetch_and_sub1) && \
 695     defined(AO_HAVE_fetch_and_sub1_write)
 696 #  define AO_fetch_and_sub1(addr) \
 697         AO_fetch_and_sub1_write(addr)
 698 #  define AO_HAVE_fetch_and_sub1
 699 #endif
 700 #if !defined(AO_HAVE_fetch_and_sub1) && \
 701     defined(AO_HAVE_fetch_and_sub1_read)
 702 #  define AO_fetch_and_sub1(addr) \
 703         AO_fetch_and_sub1_read(addr)
 704 #  define AO_HAVE_fetch_and_sub1
 705 #endif
 706
 707 #if defined(AO_HAVE_fetch_and_sub1_acquire) &&\
 708     defined(AO_HAVE_nop_full) && \
 709     !defined(AO_HAVE_fetch_and_sub1_full)
 710 #  define AO_fetch_and_sub1_full(addr) \
 711         (AO_nop_full(), AO_fetch_and_sub1_acquire(addr))
 712 #  define AO_HAVE_fetch_and_sub1_full
 713 #endif
 714
 715 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
 716     defined(AO_HAVE_fetch_and_sub1_write)
 717 #  define AO_fetch_and_sub1_release_write(addr) \
 718         AO_fetch_and_sub1_write(addr)
 719 #  define AO_HAVE_fetch_and_sub1_release_write
 720 #endif
 721 #if !defined(AO_HAVE_fetch_and_sub1_release_write) && \
 722     defined(AO_HAVE_fetch_and_sub1_release)
 723 #  define AO_fetch_and_sub1_release_write(addr) \
 724         AO_fetch_and_sub1_release(addr)
 725 #  define AO_HAVE_fetch_and_sub1_release_write
 726 #endif
 727 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
 728     defined(AO_HAVE_fetch_and_sub1_read)
 729 #  define AO_fetch_and_sub1_acquire_read(addr) \
 730         AO_fetch_and_sub1_read(addr)
 731 #  define AO_HAVE_fetch_and_sub1_acquire_read
 732 #endif
 733 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) && \
 734     defined(AO_HAVE_fetch_and_sub1_acquire)
 735 #  define AO_fetch_and_sub1_acquire_read(addr) \
 736         AO_fetch_and_sub1_acquire(addr)
 737 #  define AO_HAVE_fetch_and_sub1_acquire_read
 738 #endif
 739
 740 #ifdef AO_NO_DD_ORDERING
 741 #  if defined(AO_HAVE_fetch_and_sub1_acquire_read)
 742 #    define AO_fetch_and_sub1_dd_acquire_read(addr) \
 743         AO_fetch_and_sub1_acquire_read(addr)
 744 #    define AO_HAVE_fetch_and_sub1_dd_acquire_read
 745 #  endif
 746 #else
 747 #  if defined(AO_HAVE_fetch_and_sub1)
 748 #    define AO_fetch_and_sub1_dd_acquire_read(addr) AO_fetch_and_sub1(addr)
 749 #    define AO_HAVE_fetch_and_sub1_dd_acquire_read
 750 #  endif
 751 #endif
 752
 753 /* Atomic or */
 754 #if defined(AO_HAVE_compare_and_swap_full) && \
 755     !defined(AO_HAVE_or_full)
 756    AO_INLINE void
 757    AO_or_full(volatile AO_t *addr, AO_t incr)
 758    {
 759      AO_t old;
 760      do
 761        {
 762          old = *addr;
 763        }
 764      while (!AO_compare_and_swap_full(addr, old, (old | incr)));
 765    }
 766 #  define AO_HAVE_or_full
 767 #endif
 768
 769 #if defined(AO_HAVE_or_full)
 770 #  if !defined(AO_HAVE_or_release)
 771 #    define AO_or_release(addr, val) \
 772          AO_or_full(addr, val)
 773 #    define AO_HAVE_or_release
 774 #  endif
 775 #  if !defined(AO_HAVE_or_acquire)
 776 #    define AO_or_acquire(addr, val) \
 777          AO_or_full(addr, val)
 778 #    define AO_HAVE_or_acquire
 779 #  endif
 780 #  if !defined(AO_HAVE_or_write)
 781 #    define AO_or_write(addr, val) \
 782          AO_or_full(addr, val)
 783 #    define AO_HAVE_or_write
 784 #  endif
 785 #  if !defined(AO_HAVE_or_read)
 786 #    define AO_or_read(addr, val) \
 787          AO_or_full(addr, val)
 788 #    define AO_HAVE_or_read
 789 #  endif
 790 #endif /* AO_HAVE_or_full */
 791
 792 #if !defined(AO_HAVE_or) && \
 793     defined(AO_HAVE_or_release)
 794 #  define AO_or(addr, val) \
 795         AO_or_release(addr, val)
 796 #  define AO_HAVE_or
 797 #endif
 798 #if !defined(AO_HAVE_or) && \
 799     defined(AO_HAVE_or_acquire)
 800 #  define AO_or(addr, val) \
 801         AO_or_acquire(addr, val)
 802 #  define AO_HAVE_or
 803 #endif
 804 #if !defined(AO_HAVE_or) && \
 805     defined(AO_HAVE_or_write)
 806 #  define AO_or(addr, val) \
 807         AO_or_write(addr, val)
 808 #  define AO_HAVE_or
 809 #endif
 810 #if !defined(AO_HAVE_or) && \
 811     defined(AO_HAVE_or_read)
 812 #  define AO_or(addr, val) \
 813         AO_or_read(addr, val)
 814 #  define AO_HAVE_or
 815 #endif
 816
 817 #if defined(AO_HAVE_or_acquire) &&\
 818     defined(AO_HAVE_nop_full) && \
 819     !defined(AO_HAVE_or_full)
 820 #  define AO_or_full(addr, val) \
 821         (AO_nop_full(), AO_or_acquire(addr, val))
 822 #endif
 823
 824 #if !defined(AO_HAVE_or_release_write) && \
 825     defined(AO_HAVE_or_write)
 826 #  define AO_or_release_write(addr, val) \
 827         AO_or_write(addr, val)
 828 #  define AO_HAVE_or_release_write
 829 #endif
 830 #if !defined(AO_HAVE_or_release_write) && \
 831     defined(AO_HAVE_or_release)
 832 #  define AO_or_release_write(addr, val) \
 833         AO_or_release(addr, val)
 834 #  define AO_HAVE_or_release_write
 835 #endif
 836 #if !defined(AO_HAVE_or_acquire_read) && \
 837     defined(AO_HAVE_or_read)
 838 #  define AO_or_acquire_read(addr, val) \
 839         AO_or_read(addr, val)
 840 #  define AO_HAVE_or_acquire_read
 841 #endif
 842 #if !defined(AO_HAVE_or_acquire_read) && \
 843     defined(AO_HAVE_or_acquire)
 844 #  define AO_or_acquire_read(addr, val) \
 845         AO_or_acquire(addr, val)
 846 #  define AO_HAVE_or_acquire_read
 847 #endif
 848
 849 /* dd_aquire_read is meaningless.       */
 850
 851 /* Test_and_set */
 852
 853 #if defined(AO_HAVE_test_and_set_full)
 854 #  if !defined(AO_HAVE_test_and_set_release)
 855 #    define AO_test_and_set_release(addr) \
 856          AO_test_and_set_full(addr)
 857 #    define AO_HAVE_test_and_set_release
 858 #  endif
 859 #  if !defined(AO_HAVE_test_and_set_acquire)
 860 #    define AO_test_and_set_acquire(addr) \
 861          AO_test_and_set_full(addr)
 862 #    define AO_HAVE_test_and_set_acquire
 863 #  endif
 864 #  if !defined(AO_HAVE_test_and_set_write)
 865 #    define AO_test_and_set_write(addr) \
 866          AO_test_and_set_full(addr)
 867 #    define AO_HAVE_test_and_set_write
 868 #  endif
 869 #  if !defined(AO_HAVE_test_and_set_read)
 870 #    define AO_test_and_set_read(addr) \
 871          AO_test_and_set_full(addr)
 872 #    define AO_HAVE_test_and_set_read
 873 #  endif
 874 #endif /* AO_HAVE_test_and_set_full */
 875
 876 #if !defined(AO_HAVE_test_and_set) && \
 877     defined(AO_HAVE_test_and_set_release)
 878 #  define AO_test_and_set(addr) \
 879         AO_test_and_set_release(addr)
 880 #  define AO_HAVE_test_and_set
 881 #endif
 882 #if !defined(AO_HAVE_test_and_set) && \
 883     defined(AO_HAVE_test_and_set_acquire)
 884 #  define AO_test_and_set(addr) \
 885         AO_test_and_set_acquire(addr)
 886 #  define AO_HAVE_test_and_set
 887 #endif
 888 #if !defined(AO_HAVE_test_and_set) && \
 889     defined(AO_HAVE_test_and_set_write)
 890 #  define AO_test_and_set(addr) \
 891         AO_test_and_set_write(addr)
 892 #  define AO_HAVE_test_and_set
 893 #endif
 894 #if !defined(AO_HAVE_test_and_set) && \
 895     defined(AO_HAVE_test_and_set_read)
 896 #  define AO_test_and_set(addr) \
 897         AO_test_and_set_read(addr)
 898 #  define AO_HAVE_test_and_set
 899 #endif
 900
 901 #if defined(AO_HAVE_test_and_set_acquire) &&\
 902     defined(AO_HAVE_nop_full) && \
 903     !defined(AO_HAVE_test_and_set_full)
 904 #  define AO_test_and_set_full(addr) \
 905         (AO_nop_full(), AO_test_and_set_acquire(addr))
 906 #  define AO_HAVE_test_and_set_full
 907 #endif
 908
 909 #if !defined(AO_HAVE_test_and_set_release_write) && \
 910     defined(AO_HAVE_test_and_set_write)
 911 #  define AO_test_and_set_release_write(addr) \
 912         AO_test_and_set_write(addr)
 913 #  define AO_HAVE_test_and_set_release_write
 914 #endif
 915 #if !defined(AO_HAVE_test_and_set_release_write) && \
 916     defined(AO_HAVE_test_and_set_release)
 917 #  define AO_test_and_set_release_write(addr) \
 918         AO_test_and_set_release(addr)
 919 #  define AO_HAVE_test_and_set_release_write
 920 #endif
 921 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
 922     defined(AO_HAVE_test_and_set_read)
 923 #  define AO_test_and_set_acquire_read(addr) \
 924         AO_test_and_set_read(addr)
 925 #  define AO_HAVE_test_and_set_acquire_read
 926 #endif
 927 #if !defined(AO_HAVE_test_and_set_acquire_read) && \
 928     defined(AO_HAVE_test_and_set_acquire)
 929 #  define AO_test_and_set_acquire_read(addr) \
 930         AO_test_and_set_acquire(addr)
 931 #  define AO_HAVE_test_and_set_acquire_read
 932 #endif
 933
 934 #ifdef AO_NO_DD_ORDERING
 935 #  if defined(AO_HAVE_test_and_set_acquire_read)
 936 #    define AO_test_and_set_dd_acquire_read(addr) \
 937         AO_test_and_set_acquire_read(addr)
 938 #    define AO_HAVE_test_and_set_dd_acquire_read
 939 #  endif
 940 #else
 941 #  if defined(AO_HAVE_test_and_set)
 942 #    define AO_test_and_set_dd_acquire_read(addr) AO_test_and_set(addr)
 943 #    define AO_HAVE_test_and_set_dd_acquire_read
 944 #  endif
 945 #endif
 946
 947 /* Compare_and_swap */
 948 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
 949     && !defined(AO_HAVE_compare_and_swap_acquire)
 950    AO_INLINE int
 951    AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old, AO_t new_val)
 952    {
 953      int result = AO_compare_and_swap(addr, old, new_val);
 954      AO_nop_full();
 955      return result;
 956    }
 957 #  define AO_HAVE_compare_and_swap_acquire
 958 #endif
 959 #if defined(AO_HAVE_compare_and_swap) && defined(AO_HAVE_nop_full)\
 960     && !defined(AO_HAVE_compare_and_swap_release)
 961 #  define AO_compare_and_swap_release(addr, old, new_val) \
 962         (AO_nop_full(), AO_compare_and_swap(addr, old, new_val))
 963 #  define AO_HAVE_compare_and_swap_release
 964 #endif
 965 #if defined(AO_HAVE_compare_and_swap_full)
 966 #  if !defined(AO_HAVE_compare_and_swap_release)
 967 #    define AO_compare_and_swap_release(addr, old, new_val) \
 968          AO_compare_and_swap_full(addr, old, new_val)
 969 #    define AO_HAVE_compare_and_swap_release
 970 #  endif
 971 #  if !defined(AO_HAVE_compare_and_swap_acquire)
 972 #    define AO_compare_and_swap_acquire(addr, old, new_val) \
 973          AO_compare_and_swap_full(addr, old, new_val)
 974 #    define AO_HAVE_compare_and_swap_acquire
 975 #  endif
 976 #  if !defined(AO_HAVE_compare_and_swap_write)
 977 #    define AO_compare_and_swap_write(addr, old, new_val) \
 978          AO_compare_and_swap_full(addr, old, new_val)
 979 #    define AO_HAVE_compare_and_swap_write
 980 #  endif
 981 #  if !defined(AO_HAVE_compare_and_swap_read)
 982 #    define AO_compare_and_swap_read(addr, old, new_val) \
 983          AO_compare_and_swap_full(addr, old, new_val)
 984 #    define AO_HAVE_compare_and_swap_read
 985 #  endif
 986 #endif /* AO_HAVE_compare_and_swap_full */
 987
 988 #if !defined(AO_HAVE_compare_and_swap) && \
 989     defined(AO_HAVE_compare_and_swap_release)
 990 #  define AO_compare_and_swap(addr, old, new_val) \
 991         AO_compare_and_swap_release(addr, old, new_val)
 992 #  define AO_HAVE_compare_and_swap
 993 #endif
 994 #if !defined(AO_HAVE_compare_and_swap) && \
 995     defined(AO_HAVE_compare_and_swap_acquire)
 996 #  define AO_compare_and_swap(addr, old, new_val) \
 997         AO_compare_and_swap_acquire(addr, old, new_val)
 998 #  define AO_HAVE_compare_and_swap
 999 #endif
1000 #if !defined(AO_HAVE_compare_and_swap) && \
1001     defined(AO_HAVE_compare_and_swap_write)
1002 #  define AO_compare_and_swap(addr, old, new_val) \
1003         AO_compare_and_swap_write(addr, old, new_val)
1004 #  define AO_HAVE_compare_and_swap
1005 #endif
1006 #if !defined(AO_HAVE_compare_and_swap) && \
1007     defined(AO_HAVE_compare_and_swap_read)
1008 #  define AO_compare_and_swap(addr, old, new_val) \
1009         AO_compare_and_swap_read(addr, old, new_val)
1010 #  define AO_HAVE_compare_and_swap
1011 #endif
1012
1013 #if defined(AO_HAVE_compare_and_swap_acquire) &&\
1014     defined(AO_HAVE_nop_full) && \
1015     !defined(AO_HAVE_compare_and_swap_full)
1016 #  define AO_compare_and_swap_full(addr, old, new_val) \
1017         (AO_nop_full(), AO_compare_and_swap_acquire(addr, old, new_val))
1018 #  define AO_HAVE_compare_and_swap_full
1019 #endif
1020
1021 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
1022     defined(AO_HAVE_compare_and_swap_write)
1023 #  define AO_compare_and_swap_release_write(addr, old, new_val) \
1024         AO_compare_and_swap_write(addr, old, new_val)
1025 #  define AO_HAVE_compare_and_swap_release_write
1026 #endif
1027 #if !defined(AO_HAVE_compare_and_swap_release_write) && \
1028     defined(AO_HAVE_compare_and_swap_release)
1029 #  define AO_compare_and_swap_release_write(addr, old, new_val) \
1030         AO_compare_and_swap_release(addr, old, new_val)
1031 #  define AO_HAVE_compare_and_swap_release_write
1032 #endif
1033 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
1034     defined(AO_HAVE_compare_and_swap_read)
1035 #  define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1036         AO_compare_and_swap_read(addr, old, new_val)
1037 #  define AO_HAVE_compare_and_swap_acquire_read
1038 #endif
1039 #if !defined(AO_HAVE_compare_and_swap_acquire_read) && \
1040     defined(AO_HAVE_compare_and_swap_acquire)
1041 #  define AO_compare_and_swap_acquire_read(addr, old, new_val) \
1042         AO_compare_and_swap_acquire(addr, old, new_val)
1043 #  define AO_HAVE_compare_and_swap_acquire_read
1044 #endif
1045
1046 #ifdef AO_NO_DD_ORDERING
1047 #  if defined(AO_HAVE_compare_and_swap_acquire_read)
1048 #    define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1049         AO_compare_and_swap_acquire_read(addr, old, new_val)
1050 #    define AO_HAVE_compare_and_swap_dd_acquire_read
1051 #  endif
1052 #else
1053 #  if defined(AO_HAVE_compare_and_swap)
1054 #    define AO_compare_and_swap_dd_acquire_read(addr, old, new_val) \
1055         AO_compare_and_swap(addr, old, new_val)
1056 #    define AO_HAVE_compare_and_swap_dd_acquire_read
1057 #  endif
1058 #endif
1059
1060 #include "generalize-small.h"
1061
1062 /* Compare_double_and_swap_double */
1063 #if defined(AO_HAVE_compare_double_and_swap_double) && defined(AO_HAVE_nop_full)\
1064     && !defined(AO_HAVE_compare_double_and_swap_double_acquire)
1065    AO_INLINE int
1066    AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
1067                                              AO_t o1, AO_t o2,
1068                                              AO_t n1, AO_t n2)
1069    {
1070      int result = AO_compare_double_and_swap_double(addr, o1, o2, n1, n2);
1071      AO_nop_full();
1072      return result;
1073    }
1074 #  define AO_HAVE_compare_double_and_swap_double_acquire
1075 #endif
1076 #if defined(AO_HAVE_compare_double_and_swap_double) \
1077     && defined(AO_HAVE_nop_full)\
1078     && !defined(AO_HAVE_compare_double_and_swap_double_release)
1079 #  define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
1080         (AO_nop_full(), AO_compare_double_and_swap_double(addr, o1, o2, n1, n2))
1081 #  define AO_HAVE_compare_double_and_swap_double_release
1082 #endif
1083 #if defined(AO_HAVE_compare_double_and_swap_double_full)
1084 #  if !defined(AO_HAVE_compare_double_and_swap_double_release)
1085 #    define AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2) \
1086          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1087 #    define AO_HAVE_compare_double_and_swap_double_release
1088 #  endif
1089 #  if !defined(AO_HAVE_compare_double_and_swap_double_acquire)
1090 #    define AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2) \
1091          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1092 #    define AO_HAVE_compare_double_and_swap_double_acquire
1093 #  endif
1094 #  if !defined(AO_HAVE_compare_double_and_swap_double_write)
1095 #    define AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2) \
1096          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1097 #    define AO_HAVE_compare_double_and_swap_double_write
1098 #  endif
1099 #  if !defined(AO_HAVE_compare_double_and_swap_double_read)
1100 #    define AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2) \
1101          AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2)
1102 #    define AO_HAVE_compare_double_and_swap_double_read
1103 #  endif
1104 #endif /* AO_HAVE_compare_double_and_swap_double_full */
1105
1106 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1107     defined(AO_HAVE_compare_double_and_swap_double_release)
1108 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1109         AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
1110 #  define AO_HAVE_compare_double_and_swap_double
1111 #endif
1112 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1113     defined(AO_HAVE_compare_double_and_swap_double_acquire)
1114 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1115         AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
1116 #  define AO_HAVE_compare_double_and_swap_double
1117 #endif
1118 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1119     defined(AO_HAVE_compare_double_and_swap_double_write)
1120 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1121         AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
1122 #  define AO_HAVE_compare_double_and_swap_double
1123 #endif
1124 #if !defined(AO_HAVE_compare_double_and_swap_double) && \
1125     defined(AO_HAVE_compare_double_and_swap_double_read)
1126 #  define AO_compare_double_and_swap_double(addr, o1, o2, n1, n2) \
1127         AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
1128 #  define AO_HAVE_compare_double_and_swap_double
1129 #endif
1130
1131 #if defined(AO_HAVE_compare_double_and_swap_double_acquire) &&\
1132     defined(AO_HAVE_nop_full) && \
1133     !defined(AO_HAVE_compare_double_and_swap_double_full)
1134 #  define AO_compare_double_and_swap_double_full(addr, o1, o2, n1, n2) \
1135         (AO_nop_full(), AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2))
1136 #  define AO_HAVE_compare_double_and_swap_double_full
1137 #endif
1138
1139 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
1140     defined(AO_HAVE_compare_double_and_swap_double_write)
1141 #  define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
1142         AO_compare_double_and_swap_double_write(addr, o1, o2, n1, n2)
1143 #  define AO_HAVE_compare_double_and_swap_double_release_write
1144 #endif
1145 #if !defined(AO_HAVE_compare_double_and_swap_double_release_write) && \
1146     defined(AO_HAVE_compare_double_and_swap_double_release)
1147 #  define AO_compare_double_and_swap_double_release_write(addr, o1, o2, n1, n2) \
1148         AO_compare_double_and_swap_double_release(addr, o1, o2, n1, n2)
1149 #  define AO_HAVE_compare_double_and_swap_double_release_write
1150 #endif
1151 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
1152     defined(AO_HAVE_compare_double_and_swap_double_read)
1153 #  define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
1154         AO_compare_double_and_swap_double_read(addr, o1, o2, n1, n2)
1155 #  define AO_HAVE_compare_double_and_swap_double_acquire_read
1156 #endif
1157 #if !defined(AO_HAVE_compare_double_and_swap_double_acquire_read) && \
1158     defined(AO_HAVE_compare_double_and_swap_double_acquire)
1159 #  define AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2) \
1160         AO_compare_double_and_swap_double_acquire(addr, o1, o2, n1, n2)
1161 #  define AO_HAVE_compare_double_and_swap_double_acquire_read
1162 #endif
1163
1164 #ifdef AO_NO_DD_ORDERING
1165 #  if defined(AO_HAVE_compare_double_and_swap_double_acquire_read)
1166 #    define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
1167         AO_compare_double_and_swap_double_acquire_read(addr, o1, o2, n1, n2)
1168 #    define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
1169 #  endif
1170 #else
1171 #  if defined(AO_HAVE_compare_double_and_swap_double)
1172 #    define AO_compare_double_and_swap_double_dd_acquire_read(addr, o1, o2, n1, n2) \
1173         AO_compare_double_and_swap_double(addr, o1, o2, n1, n2)
1174 #    define AO_HAVE_compare_double_and_swap_double_dd_acquire_read
1175 #  endif
1176 #endif
1177
1178 /* Compare_and_swap_double */
1179 #if defined(AO_HAVE_compare_and_swap_double) && defined(AO_HAVE_nop_full)\
1180     && !defined(AO_HAVE_compare_and_swap_double_acquire)
1181    AO_INLINE int
1182    AO_compare_and_swap_double_acquire(volatile AO_double_t *addr,
1183                                              AO_t o1, AO_t o2,
1184                                              AO_t n1, AO_t n2)
1185    {
1186      int result = AO_compare_and_swap_double(addr, o1, n1, n2);
1187      AO_nop_full();
1188      return result;
1189    }
1190 #  define AO_HAVE_compare_and_swap_double_acquire
1191 #endif
1192 #if defined(AO_HAVE_compare_and_swap_double) \
1193     && defined(AO_HAVE_nop_full)\
1194     && !defined(AO_HAVE_compare_and_swap_double_release)
1195 #  define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
1196         (AO_nop_full(), AO_compare_and_swap_double(addr, o1, n1, n2))
1197 #  define AO_HAVE_compare_and_swap_double_release
1198 #endif
1199 #if defined(AO_HAVE_compare_and_swap_double_full)
1200 #  if !defined(AO_HAVE_compare_and_swap_double_release)
1201 #    define AO_compare_and_swap_double_release(addr, o1, n1, n2) \
1202          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1203 #    define AO_HAVE_compare_and_swap_double_release
1204 #  endif
1205 #  if !defined(AO_HAVE_compare_and_swap_double_acquire)
1206 #    define AO_compare_and_swap_double_acquire(addr, o1, n1, n2) \
1207          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1208 #    define AO_HAVE_compare_and_swap_double_acquire
1209 #  endif
1210 #  if !defined(AO_HAVE_compare_and_swap_double_write)
1211 #    define AO_compare_and_swap_double_write(addr, o1, n1, n2) \
1212          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1213 #    define AO_HAVE_compare_and_swap_double_write
1214 #  endif
1215 #  if !defined(AO_HAVE_compare_and_swap_double_read)
1216 #    define AO_compare_and_swap_double_read(addr, o1, n1, n2) \
1217          AO_compare_and_swap_double_full(addr, o1, n1, n2)
1218 #    define AO_HAVE_compare_and_swap_double_read
1219 #  endif
1220 #endif /* AO_HAVE_compare_and_swap_double_full */
1221
1222 #if !defined(AO_HAVE_compare_and_swap_double) && \
1223     defined(AO_HAVE_compare_and_swap_double_release)
1224 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1225         AO_compare_and_swap_double_release(addr, o1, n1, n2)
1226 #  define AO_HAVE_compare_and_swap_double
1227 #endif
1228 #if !defined(AO_HAVE_compare_and_swap_double) && \
1229     defined(AO_HAVE_compare_and_swap_double_acquire)
1230 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1231         AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
1232 #  define AO_HAVE_compare_and_swap_double
1233 #endif
1234 #if !defined(AO_HAVE_compare_and_swap_double) && \
1235     defined(AO_HAVE_compare_and_swap_double_write)
1236 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1237         AO_compare_and_swap_double_write(addr, o1, n1, n2)
1238 #  define AO_HAVE_compare_and_swap_double
1239 #endif
1240 #if !defined(AO_HAVE_compare_and_swap_double) && \
1241     defined(AO_HAVE_compare_and_swap_double_read)
1242 #  define AO_compare_and_swap_double(addr, o1, n1, n2) \
1243         AO_compare_and_swap_double_read(addr, o1, n1, n2)
1244 #  define AO_HAVE_compare_and_swap_double
1245 #endif
1246
1247 #if defined(AO_HAVE_compare_and_swap_double_acquire) &&\
1248     defined(AO_HAVE_nop_full) && \
1249     !defined(AO_HAVE_compare_and_swap_double_full)
1250 #  define AO_compare_and_swap_double_full(addr, o1, n1, n2) \
1251         (AO_nop_full(), AO_compare_and_swap_double_acquire(addr, o1, n1, n2))
1252 #  define AO_HAVE_compare_and_swap_double_full
1253 #endif
1254
1255 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
1256     defined(AO_HAVE_compare_and_swap_double_write)
1257 #  define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
1258         AO_compare_and_swap_double_write(addr, o1, n1, n2)
1259 #  define AO_HAVE_compare_and_swap_double_release_write
1260 #endif
1261 #if !defined(AO_HAVE_compare_and_swap_double_release_write) && \
1262     defined(AO_HAVE_compare_and_swap_double_release)
1263 #  define AO_compare_and_swap_double_release_write(addr, o1, n1, n2) \
1264         AO_compare_and_swap_double_release(addr, o1, n1, n2)
1265 #  define AO_HAVE_compare_and_swap_double_release_write
1266 #endif
1267 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
1268     defined(AO_HAVE_compare_and_swap_double_read)
1269 #  define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
1270         AO_compare_and_swap_double_read(addr, o1, n1, n2)
1271 #  define AO_HAVE_compare_and_swap_double_acquire_read
1272 #endif
1273 #if !defined(AO_HAVE_compare_and_swap_double_acquire_read) && \
1274     defined(AO_HAVE_compare_and_swap_double_acquire)
1275 #  define AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2) \
1276         AO_compare_and_swap_double_acquire(addr, o1, n1, n2)
1277 #  define AO_HAVE_compare_and_swap_double_acquire_read
1278 #endif
1279
1280 #ifdef AO_NO_DD_ORDERING
1281 #  if defined(AO_HAVE_compare_and_swap_double_acquire_read)
1282 #    define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
1283         AO_compare_and_swap_double_acquire_read(addr, o1, n1, n2)
1284 #    define AO_HAVE_compare_and_swap_double_dd_acquire_read
1285 #  endif
1286 #else
1287 #  if defined(AO_HAVE_compare_and_swap_double)
1288 #    define AO_compare_and_swap_double_dd_acquire_read(addr, o1, n1, n2) \
1289         AO_compare_and_swap_double(addr, o1, n1, n2)
1290 #    define AO_HAVE_compare_and_swap_double_dd_acquire_read
1291 #  endif
1292 #endif