mono/utils/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  * Copyright 2012 Xamarin Inc
   9  */
  10
  11 #ifndef _WAPI_ATOMIC_H_
  12 #define _WAPI_ATOMIC_H_
  13
  14 #if defined(__NetBSD__)
  15 #include <sys/param.h>
  16
  17 #if __NetBSD_Version__ > 499004000
  18 #include <sys/atomic.h>
  19 #define HAVE_ATOMIC_OPS
  20 #endif
  21
  22 #endif
  23
  24 #include <glib.h>
  25
  26 #if defined(__WIN32__) || defined(_WIN32)
  27
  28 #include <windows.h>
  29
  30 #elif defined(__NetBSD__) && defined(HAVE_ATOMIC_OPS)
  31
  32 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  33        gint32 exch, gint32 comp)
  34 {
  35        return atomic_cas_32((uint32_t*)dest, comp, exch);
  36 }
  37
  38 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  39 {
  40        return atomic_cas_ptr(dest, comp, exch);
  41 }
  42
  43 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  44 {
  45        return atomic_inc_32_nv((uint32_t*)val);
  46 }
  47
  48 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  49 {
  50        return atomic_dec_32_nv((uint32_t*)val);
  51 }
  52
  53 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  54 {
  55        return atomic_swap_32((uint32_t*)val, new_val);
  56 }
  57
  58 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
  59                gpointer new_val)
  60 {
  61        return atomic_swap_ptr(val, new_val);
  62 }
  63
  64 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
  65 {
  66        return atomic_add_32_nv((uint32_t*)val, add) - add;
  67 }
  68
  69 #elif defined(__i386__) || defined(__x86_64__)
  70
  71 /*
  72  * NB: The *Pointer() functions here assume that
  73  * sizeof(pointer)==sizeof(gint32)
  74  *
  75  * NB2: These asm functions assume 486+ (some of the opcodes dont
  76  * exist on 386).  If this becomes an issue, we can get configure to
  77  * fall back to the non-atomic C versions of these calls.
  78  */
  79
  80 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  81                                                 gint32 exch, gint32 comp)
  82 {
  83         gint32 old;
  84
  85         __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  86                               : "=m" (*dest), "=a" (old)
  87                               : "r" (exch), "m" (*dest), "a" (comp));
  88         return(old);
  89 }
  90
  91 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  92 {
  93         gpointer old;
  94
  95         __asm__ __volatile__ ("lock; "
  96 #if defined(__x86_64__)  && !defined(__native_client__)
  97                               "cmpxchgq"
  98 #else
  99                               "cmpxchgl"
 100 #endif
 101                               " %2, %0"
 102                               : "=m" (*dest), "=a" (old)
 103                               : "r" (exch), "m" (*dest), "a" (comp));
 104
 105         return(old);
 106 }
 107
 108 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 109 {
 110         gint32 tmp;
 111
 112         __asm__ __volatile__ ("lock; xaddl %0, %1"
 113                               : "=r" (tmp), "=m" (*val)
 114                               : "0" (1), "m" (*val));
 115
 116         return(tmp+1);
 117 }
 118
 119 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 120 {
 121         gint32 tmp;
 122
 123         __asm__ __volatile__ ("lock; xaddl %0, %1"
 124                               : "=r" (tmp), "=m" (*val)
 125                               : "0" (-1), "m" (*val));
 126
 127         return(tmp-1);
 128 }
 129
 130 /*
 131  * See
 132  * http://msdn.microsoft.com/msdnmag/issues/0700/Win32/
 133  * for the reasons for using cmpxchg and a loop here.
 134  */
 135 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 136 {
 137         gint32 ret;
 138
 139         __asm__ __volatile__ ("1:; lock; cmpxchgl %2, %0; jne 1b"
 140                               : "=m" (*val), "=a" (ret)
 141                               : "r" (new_val), "m" (*val), "a" (*val));
 142         return(ret);
 143 }
 144
 145 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
 146                                                   gpointer new_val)
 147 {
 148         gpointer ret;
 149
 150         __asm__ __volatile__ ("1:; lock; "
 151 #if defined(__x86_64__)  && !defined(__native_client__)
 152                               "cmpxchgq"
 153 #else
 154                               "cmpxchgl"
 155 #endif
 156                               " %2, %0; jne 1b"
 157                               : "=m" (*val), "=a" (ret)
 158                               : "r" (new_val), "m" (*val), "a" (*val));
 159
 160         return(ret);
 161 }
 162
 163 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 164 {
 165         gint32 ret;
 166
 167         __asm__ __volatile__ ("lock; xaddl %0, %1"
 168                               : "=r" (ret), "=m" (*val)
 169                               : "0" (add), "m" (*val));
 170
 171         return(ret);
 172 }
 173
 174 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
 175
 176 G_GNUC_UNUSED
 177 static inline gint32 InterlockedCompareExchange(volatile gint32 *_dest, gint32 _exch, gint32 _comp)
 178 {
 179        register volatile gint32 *dest asm("g1") = _dest;
 180        register gint32 comp asm("o4") = _comp;
 181        register gint32 exch asm("o5") = _exch;
 182
 183        __asm__ __volatile__(
 184                /* cas [%%g1], %%o4, %%o5 */
 185                ".word 0xdbe0500c"
 186                : "=r" (exch)
 187                : "0" (exch), "r" (dest), "r" (comp)
 188                : "memory");
 189
 190        return exch;
 191 }
 192
 193 G_GNUC_UNUSED
 194 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *_dest, gpointer _exch, gpointer _comp)
 195 {
 196        register volatile gpointer *dest asm("g1") = _dest;
 197        register gpointer comp asm("o4") = _comp;
 198        register gpointer exch asm("o5") = _exch;
 199
 200        __asm__ __volatile__(
 201 #ifdef SPARCV9
 202                /* casx [%%g1], %%o4, %%o5 */
 203                ".word 0xdbf0500c"
 204 #else
 205                /* cas [%%g1], %%o4, %%o5 */
 206                ".word 0xdbe0500c"
 207 #endif
 208                : "=r" (exch)
 209                : "0" (exch), "r" (dest), "r" (comp)
 210                : "memory");
 211
 212        return exch;
 213 }
 214
 215 G_GNUC_UNUSED
 216 static inline gint32 InterlockedIncrement(volatile gint32 *_dest)
 217 {
 218        register volatile gint32 *dest asm("g1") = _dest;
 219        register gint32 tmp asm("o4");
 220        register gint32 ret asm("o5");
 221
 222        __asm__ __volatile__(
 223                "1:     ld      [%%g1], %%o4\n\t"
 224                "       add     %%o4, 1, %%o5\n\t"
 225                /*      cas     [%%g1], %%o4, %%o5 */
 226                "       .word   0xdbe0500c\n\t"
 227                "       cmp     %%o4, %%o5\n\t"
 228                "       bne     1b\n\t"
 229                "        add    %%o5, 1, %%o5"
 230                : "=&r" (tmp), "=&r" (ret)
 231                : "r" (dest)
 232                : "memory", "cc");
 233
 234         return ret;
 235 }
 236
 237 G_GNUC_UNUSED
 238 static inline gint32 InterlockedDecrement(volatile gint32 *_dest)
 239 {
 240        register volatile gint32 *dest asm("g1") = _dest;
 241        register gint32 tmp asm("o4");
 242        register gint32 ret asm("o5");
 243
 244        __asm__ __volatile__(
 245                "1:     ld      [%%g1], %%o4\n\t"
 246                "       sub     %%o4, 1, %%o5\n\t"
 247                /*      cas     [%%g1], %%o4, %%o5 */
 248                "       .word   0xdbe0500c\n\t"
 249                "       cmp     %%o4, %%o5\n\t"
 250                "       bne     1b\n\t"
 251                "        sub    %%o5, 1, %%o5"
 252                : "=&r" (tmp), "=&r" (ret)
 253                : "r" (dest)
 254                : "memory", "cc");
 255
 256         return ret;
 257 }
 258
 259 G_GNUC_UNUSED
 260 static inline gint32 InterlockedExchange(volatile gint32 *_dest, gint32 exch)
 261 {
 262        register volatile gint32 *dest asm("g1") = _dest;
 263        register gint32 tmp asm("o4");
 264        register gint32 ret asm("o5");
 265
 266        __asm__ __volatile__(
 267                "1:     ld      [%%g1], %%o4\n\t"
 268                "       mov     %3, %%o5\n\t"
 269                /*      cas     [%%g1], %%o4, %%o5 */
 270                "       .word   0xdbe0500c\n\t"
 271                "       cmp     %%o4, %%o5\n\t"
 272                "       bne     1b\n\t"
 273                "        nop"
 274                : "=&r" (tmp), "=&r" (ret)
 275                : "r" (dest), "r" (exch)
 276                : "memory", "cc");
 277
 278         return ret;
 279 }
 280
 281 G_GNUC_UNUSED
 282 static inline gpointer InterlockedExchangePointer(volatile gpointer *_dest, gpointer exch)
 283 {
 284        register volatile gpointer *dest asm("g1") = _dest;
 285        register gpointer tmp asm("o4");
 286        register gpointer ret asm("o5");
 287
 288        __asm__ __volatile__(
 289 #ifdef SPARCV9
 290                "1:     ldx     [%%g1], %%o4\n\t"
 291 #else
 292                "1:     ld      [%%g1], %%o4\n\t"
 293 #endif
 294                "       mov     %3, %%o5\n\t"
 295 #ifdef SPARCV9
 296                /*      casx    [%%g1], %%o4, %%o5 */
 297                "       .word   0xdbf0500c\n\t"
 298 #else
 299                /*      cas     [%%g1], %%o4, %%o5 */
 300                "       .word   0xdbe0500c\n\t"
 301 #endif
 302                "       cmp     %%o4, %%o5\n\t"
 303                "       bne     1b\n\t"
 304                "        nop"
 305                : "=&r" (tmp), "=&r" (ret)
 306                : "r" (dest), "r" (exch)
 307                : "memory", "cc");
 308
 309         return ret;
 310 }
 311
 312 G_GNUC_UNUSED
 313 static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
 314 {
 315        register volatile gint32 *dest asm("g1") = _dest;
 316        register gint32 tmp asm("o4");
 317        register gint32 ret asm("o5");
 318
 319        __asm__ __volatile__(
 320                "1:     ld      [%%g1], %%o4\n\t"
 321                "       add     %%o4, %3, %%o5\n\t"
 322                /*      cas     [%%g1], %%o4, %%o5 */
 323                "       .word   0xdbe0500c\n\t"
 324                "       cmp     %%o4, %%o5\n\t"
 325                "       bne     1b\n\t"
 326                "        add    %%o5, %3, %%o5"
 327                : "=&r" (tmp), "=&r" (ret)
 328                : "r" (dest), "r" (add)
 329                : "memory", "cc");
 330
 331         return ret;
 332 }
 333
 334 #elif __s390__
 335
 336 static inline gint32
 337 InterlockedCompareExchange(volatile gint32 *dest,
 338                            gint32 exch, gint32 comp)
 339 {
 340         gint32 old;
 341
 342         __asm__ __volatile__ ("\tLA\t1,%0\n"
 343                               "\tLR\t%1,%3\n"
 344                               "\tCS\t%1,%2,0(1)\n"
 345                               : "+m" (*dest), "=&r" (old)
 346                               : "r" (exch), "r" (comp)
 347                               : "1", "cc");
 348         return(old);
 349 }
 350
 351 #ifndef __s390x__
 352 static inline gpointer
 353 InterlockedCompareExchangePointer(volatile gpointer *dest,
 354                            gpointer exch, gpointer comp)
 355 {
 356         gpointer old;
 357
 358         __asm__ __volatile__ ("\tLA\t1,%0\n"
 359                               "\tLR\t%1,%3\n"
 360                               "\tCS\t%1,%2,0(1)\n"
 361                               : "+m" (*dest), "=&r" (old)
 362                               : "r" (exch), "r" (comp)
 363                               : "1", "cc");
 364         return(old);
 365 }
 366 # else
 367 static inline gpointer
 368 InterlockedCompareExchangePointer(volatile gpointer *dest,
 369                                   gpointer exch,
 370                                   gpointer comp)
 371 {
 372         gpointer old;
 373
 374         __asm__ __volatile__ ("\tLA\t1,%0\n"
 375                               "\tLGR\t%1,%3\n"
 376                               "\tCSG\t%1,%2,0(1)\n"
 377                               : "+m" (*dest), "=&r" (old)
 378                               : "r" (exch), "r" (comp)
 379                               : "1", "cc");
 380
 381         return(old);
 382 }
 383 # endif
 384
 385 # ifndef __s390x__
 386 static inline gint32
 387 InterlockedIncrement(volatile gint32 *val)
 388 {
 389         gint32 tmp;
 390
 391         __asm__ __volatile__ ("\tLA\t2,%1\n"
 392                               "0:\tL\t%0,%1\n"
 393                               "\tLR\t1,%0\n"
 394                               "\tAHI\t1,1\n"
 395                               "\tCS\t%0,1,0(2)\n"
 396                               "\tJNZ\t0b\n"
 397                               "\tLR\t%0,1"
 398                               : "=r" (tmp), "+m" (*val)
 399                               : : "1", "2", "cc");
 400
 401         return(tmp);
 402 }
 403 # else
 404 static inline gint32
 405 InterlockedIncrement(volatile gint32 *val)
 406 {
 407         gint32 tmp;
 408
 409         __asm__ __volatile__ ("\tLA\t2,%1\n"
 410                               "0:\tLGF\t%0,%1\n"
 411                               "\tLGFR\t1,%0\n"
 412                               "\tAGHI\t1,1\n"
 413                               "\tCS\t%0,1,0(2)\n"
 414                               "\tJNZ\t0b\n"
 415                               "\tLGFR\t%0,1"
 416                               : "=r" (tmp), "+m" (*val)
 417                               : : "1", "2", "cc");
 418
 419         return(tmp);
 420 }
 421 # endif
 422
 423 # ifndef __s390x__
 424 static inline gint32
 425 InterlockedDecrement(volatile gint32 *val)
 426 {
 427         gint32 tmp;
 428
 429         __asm__ __volatile__ ("\tLA\t2,%1\n"
 430                               "0:\tL\t%0,%1\n"
 431                               "\tLR\t1,%0\n"
 432                               "\tAHI\t1,-1\n"
 433                               "\tCS\t%0,1,0(2)\n"
 434                               "\tJNZ\t0b\n"
 435                               "\tLR\t%0,1"
 436                               : "=r" (tmp), "+m" (*val)
 437                               : : "1", "2", "cc");
 438
 439         return(tmp);
 440 }
 441 # else
 442 static inline gint32
 443 InterlockedDecrement(volatile gint32 *val)
 444 {
 445         gint32 tmp;
 446
 447         __asm__ __volatile__ ("\tLA\t2,%1\n"
 448                               "0:\tLGF\t%0,%1\n"
 449                               "\tLGFR\t1,%0\n"
 450                               "\tAGHI\t1,-1\n"
 451                               "\tCS\t%0,1,0(2)\n"
 452                               "\tJNZ\t0b\n"
 453                               "\tLGFR\t%0,1"
 454                               : "=r" (tmp), "+m" (*val)
 455                               : : "1", "2", "cc");
 456
 457         return(tmp);
 458 }
 459 # endif
 460
 461 static inline gint32
 462 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 463 {
 464         gint32 ret;
 465
 466         __asm__ __volatile__ ("\tLA\t1,%0\n"
 467                               "0:\tL\t%1,%0\n"
 468                               "\tCS\t%1,%2,0(1)\n"
 469                               "\tJNZ\t0b"
 470                               : "+m" (*val), "=&r" (ret)
 471                               : "r" (new_val)
 472                               : "1", "cc");
 473
 474         return(ret);
 475 }
 476
 477 # ifndef __s390x__
 478 static inline gpointer
 479 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 480 {
 481         gpointer ret;
 482
 483         __asm__ __volatile__ ("\tLA\t1,%0\n"
 484                               "0:\tL\t%1,%0\n"
 485                               "\tCS\t%1,%2,0(1)\n"
 486                               "\tJNZ\t0b"
 487                               : "+m" (*val), "=&r" (ret)
 488                               : "r" (new_val)
 489                               : "1", "cc");
 490
 491         return(ret);
 492 }
 493 # else
 494 static inline gpointer
 495 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 496 {
 497         gpointer ret;
 498
 499         __asm__ __volatile__ ("\tLA\t1,%0\n"
 500                               "0:\tLG\t%1,%0\n"
 501                               "\tCSG\t%1,%2,0(1)\n"
 502                               "\tJNZ\t0b"
 503                               : "+m" (*val), "=&r" (ret)
 504                               : "r" (new_val)
 505                               : "1", "cc");
 506
 507         return(ret);
 508 }
 509 # endif
 510
 511 # ifndef __s390x__
 512 static inline gint32
 513 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 514 {
 515         gint32 ret;
 516
 517         __asm__ __volatile__ ("\tLA\t2,%1\n"
 518                               "0:\tL\t%0,%1\n"
 519                               "\tLR\t1,%0\n"
 520                               "\tAR\t1,%2\n"
 521                               "\tCS\t%0,1,0(2)\n"
 522                               "\tJNZ\t0b"
 523                               : "=&r" (ret), "+m" (*val)
 524                               : "r" (add)
 525                               : "1", "2", "cc");
 526
 527         return(ret);
 528 }
 529 # else
 530 static inline gint32
 531 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 532 {
 533         gint32 ret;
 534
 535         __asm__ __volatile__ ("\tLA\t2,%1\n"
 536                               "0:\tLGF\t%0,%1\n"
 537                               "\tLGFR\t1,%0\n"
 538                               "\tAGR\t1,%2\n"
 539                               "\tCS\t%0,1,0(2)\n"
 540                               "\tJNZ\t0b"
 541                               : "=&r" (ret), "+m" (*val)
 542                               : "r" (add)
 543                               : "1", "2", "cc");
 544
 545         return(ret);
 546 }
 547 # endif
 548
 549 #elif defined(__mono_ppc__)
 550
 551 #ifdef G_COMPILER_CODEWARRIOR
 552 static inline gint32 InterlockedIncrement(volatile register gint32 *val)
 553 {
 554         gint32 result = 0, tmp;
 555         register gint32 result = 0;
 556         register gint32 tmp;
 557
 558         asm
 559         {
 560                 @1:
 561                         lwarx   tmp, 0, val
 562                         addi    result, tmp, 1
 563                         stwcx.  result, 0, val
 564                         bne-    @1
 565         }
 566
 567         return result;
 568 }
 569
 570 static inline gint32 InterlockedDecrement(register volatile gint32 *val)
 571 {
 572         register gint32 result = 0;
 573         register gint32 tmp;
 574
 575         asm
 576         {
 577                 @1:
 578                         lwarx   tmp, 0, val
 579                         addi    result, tmp, -1
 580                         stwcx.  result, 0, val
 581                         bne-    @1
 582         }
 583
 584         return result;
 585 }
 586 #define InterlockedCompareExchangePointer(dest,exch,comp) (void*)InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp))
 587
 588 static inline gint32 InterlockedCompareExchange(volatile register gint32 *dest, register gint32 exch, register gint32 comp)
 589 {
 590         register gint32 tmp = 0;
 591
 592         asm
 593         {
 594                 @1:
 595                         lwarx   tmp, 0, dest
 596                         cmpw    tmp, comp
 597                         bne-    @2
 598                         stwcx.  exch, 0, dest
 599                         bne-    @1
 600                 @2:
 601         }
 602
 603         return tmp;
 604 }
 605 static inline gint32 InterlockedExchange(register volatile gint32 *dest, register gint32 exch)
 606 {
 607         register gint32 tmp = 0;
 608
 609         asm
 610         {
 611                 @1:
 612                         lwarx   tmp, 0, dest
 613                         stwcx.  exch, 0, dest
 614                         bne-    @1
 615         }
 616
 617         return tmp;
 618 }
 619 #define InterlockedExchangePointer(dest,exch) (void*)InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch))
 620 #else
 621
 622 #if defined(__mono_ppc64__) && !defined(__mono_ilp32__)
 623 #define LDREGX "ldarx"
 624 #define STREGCXD "stdcx."
 625 #define CMPREG "cmpd"
 626 #else
 627 #define LDREGX "lwarx"
 628 #define STREGCXD "stwcx."
 629 #define CMPREG "cmpw"
 630 #endif
 631
 632 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 633 {
 634         gint32 result = 0, tmp;
 635
 636         __asm__ __volatile__ ("\n1:\n\t"
 637                               "lwarx  %0, 0, %2\n\t"
 638                               "addi   %1, %0, 1\n\t"
 639                               "stwcx. %1, 0, %2\n\t"
 640                               "bne-   1b"
 641                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 642         return result + 1;
 643 }
 644
 645 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 646 {
 647         gint32 result = 0, tmp;
 648
 649         __asm__ __volatile__ ("\n1:\n\t"
 650                               "lwarx  %0, 0, %2\n\t"
 651                               "addi   %1, %0, -1\n\t"
 652                               "stwcx. %1, 0, %2\n\t"
 653                               "bne-   1b"
 654                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 655         return result - 1;
 656 }
 657
 658 static inline gpointer InterlockedCompareExchangePointer (volatile gpointer *dest,
 659                                                 gpointer exch, gpointer comp)
 660 {
 661         gpointer tmp = NULL;
 662
 663         __asm__ __volatile__ ("\n1:\n\t"
 664                              LDREGX " %0, 0, %1\n\t"
 665                              CMPREG " %0, %2\n\t"
 666                              "bne-    2f\n\t"
 667                              STREGCXD " %3, 0, %1\n\t"
 668                              "bne-    1b\n"
 669                              "2:"
 670                              : "=&r" (tmp)
 671                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 672         return(tmp);
 673 }
 674
 675 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
 676                                                 gint32 exch, gint32 comp) {
 677         gint32 tmp = 0;
 678
 679         __asm__ __volatile__ ("\n1:\n\t"
 680                              "lwarx   %0, 0, %1\n\t"
 681                              "cmpw    %0, %2\n\t"
 682                              "bne-    2f\n\t"
 683                              "stwcx.  %3, 0, %1\n\t"
 684                              "bne-    1b\n"
 685                              "2:"
 686                              : "=&r" (tmp)
 687                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 688         return(tmp);
 689 }
 690
 691 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 692 {
 693         gint32 tmp = 0;
 694
 695         __asm__ __volatile__ ("\n1:\n\t"
 696                               "lwarx  %0, 0, %2\n\t"
 697                               "stwcx. %3, 0, %2\n\t"
 698                               "bne    1b"
 699                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 700         return(tmp);
 701 }
 702
 703 static inline gpointer InterlockedExchangePointer (volatile gpointer *dest, gpointer exch)
 704 {
 705         gpointer tmp = NULL;
 706
 707         __asm__ __volatile__ ("\n1:\n\t"
 708                               LDREGX " %0, 0, %2\n\t"
 709                               STREGCXD " %3, 0, %2\n\t"
 710                               "bne    1b"
 711                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 712         return(tmp);
 713 }
 714
 715 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 716 {
 717         gint32 result, tmp;
 718         __asm__ __volatile__ ("\n1:\n\t"
 719                               "lwarx  %0, 0, %2\n\t"
 720                               "add    %1, %0, %3\n\t"
 721                               "stwcx. %1, 0, %2\n\t"
 722                               "bne    1b"
 723                               : "=&r" (result), "=&r" (tmp)
 724                               : "r" (dest), "r" (add) : "cc", "memory");
 725         return(result);
 726 }
 727
 728 #undef LDREGX
 729 #undef STREGCXD
 730 #undef CMPREG
 731
 732 #endif /* !G_COMPILER_CODEWARRIOR */
 733
 734 #elif defined(__arm__)
 735
 736 #ifdef __native_client__
 737 #define MASK_REGISTER(reg, cond) "bic" cond " " reg ", " reg ", #0xc0000000\n"
 738 #define NACL_ALIGN() ".align 4\n"
 739 #else
 740 #define MASK_REGISTER(reg, cond)
 741 #define NACL_ALIGN()
 742 #endif
 743
 744 /*
 745  * Atomic operations on ARM doesn't contain memory barriers, and the runtime code
 746  * depends on this, so we add them explicitly.
 747  */
 748
 749 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp)
 750 {
 751 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 752         gint32 ret, tmp;
 753         __asm__ __volatile__ (  "1:\n"
 754                                 NACL_ALIGN()
 755                                 "dmb\n"
 756                                 "mov    %0, #0\n"
 757                                 NACL_ALIGN()
 758                                 MASK_REGISTER("%2", "al")
 759                                 "ldrex %1, [%2]\n"
 760                                 "teq    %1, %3\n"
 761                                 "it eq\n"
 762                                 NACL_ALIGN()
 763                                 MASK_REGISTER("%2", "eq")
 764                                 "strexeq %0, %4, [%2]\n"
 765                                 "teq %0, #0\n"
 766                                 "bne 1b\n"
 767                                 "dmb\n"
 768                                 : "=&r" (tmp), "=&r" (ret)
 769                                 : "r" (dest), "r" (comp), "r" (exch)
 770                                 : "memory", "cc");
 771
 772         return ret;
 773 #else
 774         gint32 a, b;
 775
 776         __asm__ __volatile__ (    "0:\n\t"
 777                                   NACL_ALIGN()
 778                                   MASK_REGISTER("%2", "al")
 779                                   "ldr %1, [%2]\n\t"
 780                                   "cmp %1, %4\n\t"
 781                                   "mov %0, %1\n\t"
 782                                   "bne 1f\n\t"
 783                                   NACL_ALIGN()
 784                                   MASK_REGISTER("%2", "al")
 785                                   "swp %0, %3, [%2]\n\t"
 786                                   "cmp %0, %1\n\t"
 787                                   NACL_ALIGN()
 788                                   MASK_REGISTER("%2", "ne")
 789                                   "swpne %3, %0, [%2]\n\t"
 790                                   "bne 0b\n\t"
 791                                   "1:"
 792                                   : "=&r" (a), "=&r" (b)
 793                                   : "r" (dest), "r" (exch), "r" (comp)
 794                                   : "cc", "memory");
 795
 796         return a;
 797 #endif
 798 }
 799
 800 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
 801 {
 802 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 803         gpointer ret, tmp;
 804         __asm__ __volatile__ (
 805                                 "dmb\n"
 806                                 "1:\n"
 807                                 NACL_ALIGN()
 808                                 "mov    %0, #0\n"
 809                                 NACL_ALIGN()
 810                                 MASK_REGISTER("%2", "al")
 811                                 "ldrex %1, [%2]\n"
 812                                 "teq    %1, %3\n"
 813                                 "it eq\n"
 814                                 NACL_ALIGN()
 815                                 MASK_REGISTER("%2", "eq")
 816                                 "strexeq %0, %4, [%2]\n"
 817                                 "teq %0, #0\n"
 818                                 "bne 1b\n"
 819                                 "dmb\n"
 820                                 : "=&r" (tmp), "=&r" (ret)
 821                                 : "r" (dest), "r" (comp), "r" (exch)
 822                                 : "memory", "cc");
 823
 824         return ret;
 825 #else
 826         gpointer a, b;
 827
 828         __asm__ __volatile__ (    "0:\n\t"
 829                                   NACL_ALIGN()
 830                                   MASK_REGISTER("%2", "al")
 831                                   "ldr %1, [%2]\n\t"
 832                                   "cmp %1, %4\n\t"
 833                                   "mov %0, %1\n\t"
 834                                   "bne 1f\n\t"
 835                                   NACL_ALIGN()
 836                                   MASK_REGISTER("%2", "eq")
 837                                   "swpeq %0, %3, [%2]\n\t"
 838                                   "cmp %0, %1\n\t"
 839                                   NACL_ALIGN()
 840                                   MASK_REGISTER("%2", "ne")
 841                                   "swpne %3, %0, [%2]\n\t"
 842                                   "bne 0b\n\t"
 843                                   "1:"
 844                                   : "=&r" (a), "=&r" (b)
 845                                   : "r" (dest), "r" (exch), "r" (comp)
 846                                   : "cc", "memory");
 847
 848         return a;
 849 #endif
 850 }
 851
 852 static inline gint32 InterlockedIncrement(volatile gint32 *dest)
 853 {
 854 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 855         gint32 ret, flag;
 856         __asm__ __volatile__ (
 857                                 "dmb\n"
 858                                 "1:\n"
 859                                 NACL_ALIGN()
 860                                 MASK_REGISTER("%2", "al")
 861                                 "ldrex %0, [%2]\n"
 862                                 "add %0, %0, %3\n"
 863                                 NACL_ALIGN()
 864                                 MASK_REGISTER("%2", "al")
 865                                 "strex %1, %0, [%2]\n"
 866                                 "teq %1, #0\n"
 867                                 "bne 1b\n"
 868                                 "dmb\n"
 869                                 : "=&r" (ret), "=&r" (flag)
 870                                 : "r" (dest), "r" (1)
 871                                 : "memory", "cc");
 872
 873         return ret;
 874 #else
 875         gint32 a, b, c;
 876
 877         __asm__ __volatile__ (  "0:\n\t"
 878                                 NACL_ALIGN()
 879                                 MASK_REGISTER("%3", "al")
 880                                 "ldr %0, [%3]\n\t"
 881                                 "add %1, %0, %4\n\t"
 882                                 NACL_ALIGN()
 883                                 MASK_REGISTER("%3", "al")
 884                                 "swp %2, %1, [%3]\n\t"
 885                                 "cmp %0, %2\n\t"
 886                                 NACL_ALIGN()
 887                                 MASK_REGISTER("%3", "ne")
 888                                 "swpne %1, %2, [%3]\n\t"
 889                                 "bne 0b"
 890                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 891                                 : "r" (dest), "r" (1)
 892                                 : "cc", "memory");
 893
 894         return b;
 895 #endif
 896 }
 897
 898 static inline gint32 InterlockedDecrement(volatile gint32 *dest)
 899 {
 900 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 901         gint32 ret, flag;
 902         __asm__ __volatile__ (
 903                                 "dmb\n"
 904                                 "1:\n"
 905                                 NACL_ALIGN()
 906                                 MASK_REGISTER("%2", "al")
 907                                 "ldrex %0, [%2]\n"
 908                                 "sub %0, %0, %3\n"
 909                                 NACL_ALIGN()
 910                                 MASK_REGISTER("%2", "al")
 911                                 "strex %1, %0, [%2]\n"
 912                                 "teq %1, #0\n"
 913                                 "bne 1b\n"
 914                                 "dmb\n"
 915                                 : "=&r" (ret), "=&r" (flag)
 916                                 : "r" (dest), "r" (1)
 917                                 : "memory", "cc");
 918
 919         return ret;
 920 #else
 921         gint32 a, b, c;
 922
 923         __asm__ __volatile__ (  "0:\n\t"
 924                                 NACL_ALIGN()
 925                                 MASK_REGISTER("%3", "al")
 926                                 "ldr %0, [%3]\n\t"
 927                                 "add %1, %0, %4\n\t"
 928                                 NACL_ALIGN()
 929                                 MASK_REGISTER("%3", "al")
 930                                 "swp %2, %1, [%3]\n\t"
 931                                 "cmp %0, %2\n\t"
 932                                 NACL_ALIGN()
 933                                 MASK_REGISTER("%3", "ne")
 934                                 "swpne %1, %2, [%3]\n\t"
 935                                 "bne 0b"
 936                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 937                                 : "r" (dest), "r" (-1)
 938                                 : "cc", "memory");
 939
 940         return b;
 941 #endif
 942 }
 943
 944 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 945 {
 946 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 947         gint32 ret, flag;
 948         __asm__ __volatile__ (
 949                                   "dmb\n"
 950                               "1:\n"
 951                               NACL_ALIGN()
 952                               MASK_REGISTER("%3", "al")
 953                               "ldrex %0, [%3]\n"
 954                               NACL_ALIGN()
 955                               MASK_REGISTER("%3", "al")
 956                               "strex %1, %2, [%3]\n"
 957                               "teq %1, #0\n"
 958                               "bne 1b\n"
 959                                   "dmb\n"
 960                               : "=&r" (ret), "=&r" (flag)
 961                               : "r" (exch), "r" (dest)
 962                               : "memory", "cc");
 963         return ret;
 964 #else
 965         gint32 a;
 966
 967         __asm__ __volatile__ (  NACL_ALIGN()
 968                                 MASK_REGISTER("%1", "al")
 969                                 "swp %0, %2, [%1]"
 970                                 : "=&r" (a)
 971                                 : "r" (dest), "r" (exch));
 972
 973         return a;
 974 #endif
 975 }
 976
 977 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 978 {
 979 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 980         gpointer ret, flag;
 981         __asm__ __volatile__ (
 982                                   "dmb\n"
 983                               "1:\n"
 984                               NACL_ALIGN()
 985                               MASK_REGISTER("%3", "al")
 986                               "ldrex %0, [%3]\n"
 987                               NACL_ALIGN()
 988                               MASK_REGISTER("%3", "al")
 989                               "strex %1, %2, [%3]\n"
 990                               "teq %1, #0\n"
 991                               "bne 1b\n"
 992                                   "dmb\n"
 993                               : "=&r" (ret), "=&r" (flag)
 994                               : "r" (exch), "r" (dest)
 995                               : "memory", "cc");
 996         return ret;
 997 #else
 998         gpointer a;
 999
1000         __asm__ __volatile__ (  NACL_ALIGN()
1001                                 MASK_REGISTER("%1", "al")
1002                                 "swp %0, %2, [%1]"
1003                                 : "=&r" (a)
1004                                 : "r" (dest), "r" (exch));
1005
1006         return a;
1007 #endif
1008 }
1009
1010 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
1011 {
1012 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
1013         gint32 ret, tmp, flag;
1014         __asm__ __volatile__ (
1015                                 "dmb\n"
1016                                 "1:\n"
1017                                 NACL_ALIGN()
1018                                 MASK_REGISTER("%3", "al")
1019                                 "ldrex %0, [%3]\n"
1020                                 "add %1, %0, %4\n"
1021                                 NACL_ALIGN()
1022                                 MASK_REGISTER("%3", "al")
1023                                 "strex %2, %1, [%3]\n"
1024                                 "teq %2, #0\n"
1025                                 "bne 1b\n"
1026                                 "dmb\n"
1027                                 : "=&r" (ret), "=&r" (tmp), "=&r" (flag)
1028                                 : "r" (dest), "r" (add)
1029                                 : "memory", "cc");
1030
1031         return ret;
1032 #else
1033         int a, b, c;
1034
1035         __asm__ __volatile__ (  "0:\n\t"
1036                                 NACL_ALIGN()
1037                                 MASK_REGISTER("%3", "al")
1038                                 "ldr %0, [%3]\n\t"
1039                                 "add %1, %0, %4\n\t"
1040                                 NACL_ALIGN()
1041                                 MASK_REGISTER("%3", "al")
1042                                 "swp %2, %1, [%3]\n\t"
1043                                 "cmp %0, %2\n\t"
1044                                 NACL_ALIGN()
1045                                 MASK_REGISTER("%3", "ne")
1046                                 "swpne %1, %2, [%3]\n\t"
1047                                 "bne 0b"
1048                                 : "=&r" (a), "=&r" (b), "=&r" (c)
1049                                 : "r" (dest), "r" (add)
1050                                 : "cc", "memory");
1051
1052         return a;
1053 #endif
1054 }
1055
1056 #elif defined(__ia64__)
1057
1058 #ifdef __INTEL_COMPILER
1059 #include <ia64intrin.h>
1060 #endif
1061
1062 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
1063                                                 gint32 exch, gint32 comp)
1064 {
1065         gint32 old;
1066         guint64 real_comp;
1067
1068 #ifdef __INTEL_COMPILER
1069         old = _InterlockedCompareExchange (dest, exch, comp);
1070 #else
1071         /* cmpxchg4 zero extends the value read from memory */
1072         real_comp = (guint64)(guint32)comp;
1073         asm volatile ("mov ar.ccv = %2 ;;\n\t"
1074                                   "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
1075                                   : "=r" (old) : "r" (dest), "r" (real_comp), "r" (exch));
1076 #endif
1077
1078         return(old);
1079 }
1080
1081 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
1082                                                 gpointer exch, gpointer comp)
1083 {
1084         gpointer old;
1085
1086 #ifdef __INTEL_COMPILER
1087         old = _InterlockedCompareExchangePointer (dest, exch, comp);
1088 #else
1089         asm volatile ("mov ar.ccv = %2 ;;\n\t"
1090                                   "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
1091                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
1092 #endif
1093
1094         return(old);
1095 }
1096
1097 static inline gint32 InterlockedIncrement(gint32 volatile *val)
1098 {
1099 #ifdef __INTEL_COMPILER
1100         return _InterlockedIncrement (val);
1101 #else
1102         gint32 old;
1103
1104         do {
1105                 old = *val;
1106         } while (InterlockedCompareExchange (val, old + 1, old) != old);
1107
1108         return old + 1;
1109 #endif
1110 }
1111
1112 static inline gint32 InterlockedDecrement(gint32 volatile *val)
1113 {
1114 #ifdef __INTEL_COMPILER
1115         return _InterlockedDecrement (val);
1116 #else
1117         gint32 old;
1118
1119         do {
1120                 old = *val;
1121         } while (InterlockedCompareExchange (val, old - 1, old) != old);
1122
1123         return old - 1;
1124 #endif
1125 }
1126
1127 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
1128 {
1129 #ifdef __INTEL_COMPILER
1130         return _InterlockedExchange (dest, new_val);
1131 #else
1132         gint32 res;
1133
1134         do {
1135                 res = *dest;
1136         } while (InterlockedCompareExchange (dest, new_val, res) != res);
1137
1138         return res;
1139 #endif
1140 }
1141
1142 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
1143 {
1144 #ifdef __INTEL_COMPILER
1145         return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
1146 #else
1147         gpointer res;
1148
1149         do {
1150                 res = *dest;
1151         } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
1152
1153         return res;
1154 #endif
1155 }
1156
1157 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
1158 {
1159         gint32 old;
1160
1161 #ifdef __INTEL_COMPILER
1162         old = _InterlockedExchangeAdd (val, add);
1163 #else
1164         do {
1165                 old = *val;
1166         } while (InterlockedCompareExchange (val, old + add, old) != old);
1167
1168         return old;
1169 #endif
1170 }
1171
1172 #elif defined(__mips__)
1173
1174 #if SIZEOF_REGISTER == 8
1175 #error "Not implemented."
1176 #endif
1177
1178 static inline gint32 InterlockedIncrement(volatile gint32 *val)
1179 {
1180         gint32 tmp, result = 0;
1181
1182         __asm__ __volatile__ ("    .set    mips32\n"
1183                               "1:  ll      %0, %2\n"
1184                               "    addu    %1, %0, 1\n"
1185                               "    sc      %1, %2\n"
1186                               "    beqz    %1, 1b\n"
1187                               "    .set    mips0\n"
1188                               : "=&r" (result), "=&r" (tmp), "=m" (*val)
1189                               : "m" (*val));
1190         return result + 1;
1191 }
1192
1193 static inline gint32 InterlockedDecrement(volatile gint32 *val)
1194 {
1195         gint32 tmp, result = 0;
1196
1197         __asm__ __volatile__ ("    .set    mips32\n"
1198                               "1:  ll      %0, %2\n"
1199                               "    subu    %1, %0, 1\n"
1200                               "    sc      %1, %2\n"
1201                               "    beqz    %1, 1b\n"
1202                               "    .set    mips0\n"
1203                               : "=&r" (result), "=&r" (tmp), "=m" (*val)
1204                               : "m" (*val));
1205         return result - 1;
1206 }
1207
1208 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
1209                                                 gint32 exch, gint32 comp) {
1210         gint32 old, tmp;
1211
1212         __asm__ __volatile__ ("    .set    mips32\n"
1213                               "1:  ll      %0, %2\n"
1214                               "    bne     %0, %5, 2f\n"
1215                               "    move    %1, %4\n"
1216                               "    sc      %1, %2\n"
1217                               "    beqz    %1, 1b\n"
1218                               "2:  .set    mips0\n"
1219                               : "=&r" (old), "=&r" (tmp), "=m" (*dest)
1220                               : "m" (*dest), "r" (exch), "r" (comp));
1221         return(old);
1222 }
1223
1224 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
1225 {
1226         return (gpointer)(InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp)));
1227 }
1228
1229 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
1230 {
1231         gint32 result, tmp;
1232
1233         __asm__ __volatile__ ("    .set    mips32\n"
1234                               "1:  ll      %0, %2\n"
1235                               "    move    %1, %4\n"
1236                               "    sc      %1, %2\n"
1237                               "    beqz    %1, 1b\n"
1238                               "    .set    mips0\n"
1239                               : "=&r" (result), "=&r" (tmp), "=m" (*dest)
1240                               : "m" (*dest), "r" (exch));
1241         return(result);
1242 }
1243
1244 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
1245 {
1246         return (gpointer)InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch));
1247 }
1248
1249 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
1250 {
1251         gint32 result, tmp;
1252
1253         __asm__ __volatile__ ("    .set    mips32\n"
1254                               "1:  ll      %0, %2\n"
1255                               "    addu    %1, %0, %4\n"
1256                               "    sc      %1, %2\n"
1257                               "    beqz    %1, 1b\n"
1258                               "    .set    mips0\n"
1259                               : "=&r" (result), "=&r" (tmp), "=m" (*dest)
1260                               : "m" (*dest), "r" (add));
1261         return result;
1262 }
1263
1264 #else
1265
1266 #define WAPI_NO_ATOMIC_ASM
1267
1268 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
1269 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
1270 extern gint32 InterlockedIncrement(volatile gint32 *dest);
1271 extern gint32 InterlockedDecrement(volatile gint32 *dest);
1272 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
1273 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
1274 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
1275
1276 #endif
1277
1278 /* Not yet used */
1279 #ifdef USE_GCC_ATOMIC_OPS
1280
1281 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
1282                                                 gint32 exch, gint32 comp)
1283 {
1284         return __sync_val_compare_and_swap (dest, comp, exch);
1285 }
1286
1287 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
1288 {
1289         return __sync_val_compare_and_swap (dest, comp, exch);
1290 }
1291
1292 static inline gint32 InterlockedIncrement(volatile gint32 *val)
1293 {
1294         return __sync_add_and_fetch (val, 1);
1295 }
1296
1297 static inline gint32 InterlockedDecrement(volatile gint32 *val)
1298 {
1299         return __sync_add_and_fetch (val, -1);
1300 }
1301
1302 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
1303 {
1304         gint32 old_val;
1305         do {
1306                 old_val = *val;
1307         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
1308         return old_val;
1309 }
1310
1311 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
1312                                                   gpointer new_val)
1313 {
1314         gpointer old_val;
1315         do {
1316                 old_val = *val;
1317         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
1318         return old_val;
1319 }
1320
1321 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
1322 {
1323         return __sync_fetch_and_add (val, add);
1324 }
1325 #endif
1326
1327 #endif /* _WAPI_ATOMIC_H_ */