mono/utils/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  * Copyright 2012 Xamarin Inc
   9  */
  10
  11 #ifndef _WAPI_ATOMIC_H_
  12 #define _WAPI_ATOMIC_H_
  13
  14 #if defined(__NetBSD__)
  15 #include <sys/param.h>
  16
  17 #if __NetBSD_Version__ > 499004000
  18 #include <sys/atomic.h>
  19 #define HAVE_ATOMIC_OPS
  20 #endif
  21
  22 #endif
  23
  24 #include <glib.h>
  25
  26 #if defined(__WIN32__) || defined(_WIN32)
  27
  28 #include <windows.h>
  29
  30 #elif defined(__NetBSD__) && defined(HAVE_ATOMIC_OPS)
  31
  32 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  33        gint32 exch, gint32 comp)
  34 {
  35        return atomic_cas_32((uint32_t*)dest, comp, exch);
  36 }
  37
  38 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  39 {
  40        return atomic_cas_ptr(dest, comp, exch);
  41 }
  42
  43 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  44 {
  45        return atomic_inc_32_nv((uint32_t*)val);
  46 }
  47
  48 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  49 {
  50        return atomic_dec_32_nv((uint32_t*)val);
  51 }
  52
  53 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  54 {
  55        return atomic_swap_32((uint32_t*)val, new_val);
  56 }
  57
  58 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
  59                gpointer new_val)
  60 {
  61        return atomic_swap_ptr(val, new_val);
  62 }
  63
  64 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
  65 {
  66        return atomic_add_32_nv((uint32_t*)val, add) - add;
  67 }
  68
  69 #elif defined(__i386__) || defined(__x86_64__)
  70
  71 /*
  72  * NB: The *Pointer() functions here assume that
  73  * sizeof(pointer)==sizeof(gint32)
  74  *
  75  * NB2: These asm functions assume 486+ (some of the opcodes dont
  76  * exist on 386).  If this becomes an issue, we can get configure to
  77  * fall back to the non-atomic C versions of these calls.
  78  */
  79
  80 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  81                                                 gint32 exch, gint32 comp)
  82 {
  83         gint32 old;
  84
  85         __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  86                               : "=m" (*dest), "=a" (old)
  87                               : "r" (exch), "m" (*dest), "a" (comp));
  88         return(old);
  89 }
  90
  91 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  92 {
  93         gpointer old;
  94
  95         __asm__ __volatile__ ("lock; "
  96 #if defined(__x86_64__)  && !defined(__native_client__)
  97                               "cmpxchgq"
  98 #else
  99                               "cmpxchgl"
 100 #endif
 101                               " %2, %0"
 102                               : "=m" (*dest), "=a" (old)
 103                               : "r" (exch), "m" (*dest), "a" (comp));
 104
 105         return(old);
 106 }
 107
 108 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 109 {
 110         gint32 tmp;
 111
 112         __asm__ __volatile__ ("lock; xaddl %0, %1"
 113                               : "=r" (tmp), "=m" (*val)
 114                               : "0" (1), "m" (*val));
 115
 116         return(tmp+1);
 117 }
 118
 119 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 120 {
 121         gint32 tmp;
 122
 123         __asm__ __volatile__ ("lock; xaddl %0, %1"
 124                               : "=r" (tmp), "=m" (*val)
 125                               : "0" (-1), "m" (*val));
 126
 127         return(tmp-1);
 128 }
 129
 130 /*
 131  * See
 132  * http://msdn.microsoft.com/msdnmag/issues/0700/Win32/
 133  * for the reasons for using cmpxchg and a loop here.
 134  */
 135 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 136 {
 137         gint32 ret;
 138
 139         __asm__ __volatile__ ("1:; lock; cmpxchgl %2, %0; jne 1b"
 140                               : "=m" (*val), "=a" (ret)
 141                               : "r" (new_val), "m" (*val), "a" (*val));
 142         return(ret);
 143 }
 144
 145 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
 146                                                   gpointer new_val)
 147 {
 148         gpointer ret;
 149
 150         __asm__ __volatile__ ("1:; lock; "
 151 #if defined(__x86_64__)  && !defined(__native_client__)
 152                               "cmpxchgq"
 153 #else
 154                               "cmpxchgl"
 155 #endif
 156                               " %2, %0; jne 1b"
 157                               : "=m" (*val), "=a" (ret)
 158                               : "r" (new_val), "m" (*val), "a" (*val));
 159
 160         return(ret);
 161 }
 162
 163 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 164 {
 165         gint32 ret;
 166
 167         __asm__ __volatile__ ("lock; xaddl %0, %1"
 168                               : "=r" (ret), "=m" (*val)
 169                               : "0" (add), "m" (*val));
 170
 171         return(ret);
 172 }
 173
 174 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
 175
 176 G_GNUC_UNUSED
 177 static inline gint32 InterlockedCompareExchange(volatile gint32 *_dest, gint32 _exch, gint32 _comp)
 178 {
 179        register volatile gint32 *dest asm("g1") = _dest;
 180        register gint32 comp asm("o4") = _comp;
 181        register gint32 exch asm("o5") = _exch;
 182
 183        __asm__ __volatile__(
 184                /* cas [%%g1], %%o4, %%o5 */
 185                ".word 0xdbe0500c"
 186                : "=r" (exch)
 187                : "0" (exch), "r" (dest), "r" (comp)
 188                : "memory");
 189
 190        return exch;
 191 }
 192
 193 G_GNUC_UNUSED
 194 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *_dest, gpointer _exch, gpointer _comp)
 195 {
 196        register volatile gpointer *dest asm("g1") = _dest;
 197        register gpointer comp asm("o4") = _comp;
 198        register gpointer exch asm("o5") = _exch;
 199
 200        __asm__ __volatile__(
 201 #ifdef SPARCV9
 202                /* casx [%%g1], %%o4, %%o5 */
 203                ".word 0xdbf0500c"
 204 #else
 205                /* cas [%%g1], %%o4, %%o5 */
 206                ".word 0xdbe0500c"
 207 #endif
 208                : "=r" (exch)
 209                : "0" (exch), "r" (dest), "r" (comp)
 210                : "memory");
 211
 212        return exch;
 213 }
 214
 215 G_GNUC_UNUSED
 216 static inline gint32 InterlockedIncrement(volatile gint32 *_dest)
 217 {
 218        register volatile gint32 *dest asm("g1") = _dest;
 219        register gint32 tmp asm("o4");
 220        register gint32 ret asm("o5");
 221
 222        __asm__ __volatile__(
 223                "1:     ld      [%%g1], %%o4\n\t"
 224                "       add     %%o4, 1, %%o5\n\t"
 225                /*      cas     [%%g1], %%o4, %%o5 */
 226                "       .word   0xdbe0500c\n\t"
 227                "       cmp     %%o4, %%o5\n\t"
 228                "       bne     1b\n\t"
 229                "        add    %%o5, 1, %%o5"
 230                : "=&r" (tmp), "=&r" (ret)
 231                : "r" (dest)
 232                : "memory", "cc");
 233
 234         return ret;
 235 }
 236
 237 G_GNUC_UNUSED
 238 static inline gint32 InterlockedDecrement(volatile gint32 *_dest)
 239 {
 240        register volatile gint32 *dest asm("g1") = _dest;
 241        register gint32 tmp asm("o4");
 242        register gint32 ret asm("o5");
 243
 244        __asm__ __volatile__(
 245                "1:     ld      [%%g1], %%o4\n\t"
 246                "       sub     %%o4, 1, %%o5\n\t"
 247                /*      cas     [%%g1], %%o4, %%o5 */
 248                "       .word   0xdbe0500c\n\t"
 249                "       cmp     %%o4, %%o5\n\t"
 250                "       bne     1b\n\t"
 251                "        sub    %%o5, 1, %%o5"
 252                : "=&r" (tmp), "=&r" (ret)
 253                : "r" (dest)
 254                : "memory", "cc");
 255
 256         return ret;
 257 }
 258
 259 G_GNUC_UNUSED
 260 static inline gint32 InterlockedExchange(volatile gint32 *_dest, gint32 exch)
 261 {
 262        register volatile gint32 *dest asm("g1") = _dest;
 263        register gint32 tmp asm("o4");
 264        register gint32 ret asm("o5");
 265
 266        __asm__ __volatile__(
 267                "1:     ld      [%%g1], %%o4\n\t"
 268                "       mov     %3, %%o5\n\t"
 269                /*      cas     [%%g1], %%o4, %%o5 */
 270                "       .word   0xdbe0500c\n\t"
 271                "       cmp     %%o4, %%o5\n\t"
 272                "       bne     1b\n\t"
 273                "        nop"
 274                : "=&r" (tmp), "=&r" (ret)
 275                : "r" (dest), "r" (exch)
 276                : "memory", "cc");
 277
 278         return ret;
 279 }
 280
 281 G_GNUC_UNUSED
 282 static inline gpointer InterlockedExchangePointer(volatile gpointer *_dest, gpointer exch)
 283 {
 284        register volatile gpointer *dest asm("g1") = _dest;
 285        register gpointer tmp asm("o4");
 286        register gpointer ret asm("o5");
 287
 288        __asm__ __volatile__(
 289 #ifdef SPARCV9
 290                "1:     ldx     [%%g1], %%o4\n\t"
 291 #else
 292                "1:     ld      [%%g1], %%o4\n\t"
 293 #endif
 294                "       mov     %3, %%o5\n\t"
 295 #ifdef SPARCV9
 296                /*      casx    [%%g1], %%o4, %%o5 */
 297                "       .word   0xdbf0500c\n\t"
 298 #else
 299                /*      cas     [%%g1], %%o4, %%o5 */
 300                "       .word   0xdbe0500c\n\t"
 301 #endif
 302                "       cmp     %%o4, %%o5\n\t"
 303                "       bne     1b\n\t"
 304                "        nop"
 305                : "=&r" (tmp), "=&r" (ret)
 306                : "r" (dest), "r" (exch)
 307                : "memory", "cc");
 308
 309         return ret;
 310 }
 311
 312 G_GNUC_UNUSED
 313 static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
 314 {
 315        register volatile gint32 *dest asm("g1") = _dest;
 316        register gint32 tmp asm("o4");
 317        register gint32 ret asm("o5");
 318
 319        __asm__ __volatile__(
 320                "1:     ld      [%%g1], %%o4\n\t"
 321                "       add     %%o4, %3, %%o5\n\t"
 322                /*      cas     [%%g1], %%o4, %%o5 */
 323                "       .word   0xdbe0500c\n\t"
 324                "       cmp     %%o4, %%o5\n\t"
 325                "       bne     1b\n\t"
 326                "        add    %%o5, %3, %%o5"
 327                : "=&r" (tmp), "=&r" (ret)
 328                : "r" (dest), "r" (add)
 329                : "memory", "cc");
 330
 331         return ret;
 332 }
 333
 334 #elif __s390__
 335
 336 static inline gint32
 337 InterlockedCompareExchange(volatile gint32 *dest,
 338                            gint32 exch, gint32 comp)
 339 {
 340         gint32 old;
 341
 342         __asm__ __volatile__ ("\tLA\t1,%0\n"
 343                               "\tLR\t%1,%3\n"
 344                               "\tCS\t%1,%2,0(1)\n"
 345                               : "+m" (*dest), "=&r" (old)
 346                               : "r" (exch), "r" (comp)
 347                               : "1", "cc");
 348         return(old);
 349 }
 350
 351 #ifndef __s390x__
 352 static inline gpointer
 353 InterlockedCompareExchangePointer(volatile gpointer *dest,
 354                            gpointer exch, gpointer comp)
 355 {
 356         gpointer old;
 357
 358         __asm__ __volatile__ ("\tLA\t1,%0\n"
 359                               "\tLR\t%1,%3\n"
 360                               "\tCS\t%1,%2,0(1)\n"
 361                               : "+m" (*dest), "=&r" (old)
 362                               : "r" (exch), "r" (comp)
 363                               : "1", "cc");
 364         return(old);
 365 }
 366 # else
 367 static inline gpointer
 368 InterlockedCompareExchangePointer(volatile gpointer *dest,
 369                                   gpointer exch,
 370                                   gpointer comp)
 371 {
 372         gpointer old;
 373
 374         __asm__ __volatile__ ("\tLA\t1,%0\n"
 375                               "\tLGR\t%1,%3\n"
 376                               "\tCSG\t%1,%2,0(1)\n"
 377                               : "+m" (*dest), "=&r" (old)
 378                               : "r" (exch), "r" (comp)
 379                               : "1", "cc");
 380
 381         return(old);
 382 }
 383 # endif
 384
 385 # ifndef __s390x__
 386 static inline gint32
 387 InterlockedIncrement(volatile gint32 *val)
 388 {
 389         gint32 tmp;
 390
 391         __asm__ __volatile__ ("\tLA\t2,%1\n"
 392                               "0:\tL\t%0,%1\n"
 393                               "\tLR\t1,%0\n"
 394                               "\tAHI\t1,1\n"
 395                               "\tCS\t%0,1,0(2)\n"
 396                               "\tJNZ\t0b\n"
 397                               "\tLR\t%0,1"
 398                               : "=r" (tmp), "+m" (*val)
 399                               : : "1", "2", "cc");
 400
 401         return(tmp);
 402 }
 403 # else
 404 static inline gint32
 405 InterlockedIncrement(volatile gint32 *val)
 406 {
 407         gint32 tmp;
 408
 409         __asm__ __volatile__ ("\tLA\t2,%1\n"
 410                               "0:\tLGF\t%0,%1\n"
 411                               "\tLGFR\t1,%0\n"
 412                               "\tAGHI\t1,1\n"
 413                               "\tCS\t%0,1,0(2)\n"
 414                               "\tJNZ\t0b\n"
 415                               "\tLGFR\t%0,1"
 416                               : "=r" (tmp), "+m" (*val)
 417                               : : "1", "2", "cc");
 418
 419         return(tmp);
 420 }
 421 # endif
 422
 423 # ifndef __s390x__
 424 static inline gint32
 425 InterlockedDecrement(volatile gint32 *val)
 426 {
 427         gint32 tmp;
 428
 429         __asm__ __volatile__ ("\tLA\t2,%1\n"
 430                               "0:\tL\t%0,%1\n"
 431                               "\tLR\t1,%0\n"
 432                               "\tAHI\t1,-1\n"
 433                               "\tCS\t%0,1,0(2)\n"
 434                               "\tJNZ\t0b\n"
 435                               "\tLR\t%0,1"
 436                               : "=r" (tmp), "+m" (*val)
 437                               : : "1", "2", "cc");
 438
 439         return(tmp);
 440 }
 441 # else
 442 static inline gint32
 443 InterlockedDecrement(volatile gint32 *val)
 444 {
 445         gint32 tmp;
 446
 447         __asm__ __volatile__ ("\tLA\t2,%1\n"
 448                               "0:\tLGF\t%0,%1\n"
 449                               "\tLGFR\t1,%0\n"
 450                               "\tAGHI\t1,-1\n"
 451                               "\tCS\t%0,1,0(2)\n"
 452                               "\tJNZ\t0b\n"
 453                               "\tLGFR\t%0,1"
 454                               : "=r" (tmp), "+m" (*val)
 455                               : : "1", "2", "cc");
 456
 457         return(tmp);
 458 }
 459 # endif
 460
 461 static inline gint32
 462 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 463 {
 464         gint32 ret;
 465
 466         __asm__ __volatile__ ("\tLA\t1,%0\n"
 467                               "0:\tL\t%1,%0\n"
 468                               "\tCS\t%1,%2,0(1)\n"
 469                               "\tJNZ\t0b"
 470                               : "+m" (*val), "=&r" (ret)
 471                               : "r" (new_val)
 472                               : "1", "cc");
 473
 474         return(ret);
 475 }
 476
 477 # ifndef __s390x__
 478 static inline gpointer
 479 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 480 {
 481         gpointer ret;
 482
 483         __asm__ __volatile__ ("\tLA\t1,%0\n"
 484                               "0:\tL\t%1,%0\n"
 485                               "\tCS\t%1,%2,0(1)\n"
 486                               "\tJNZ\t0b"
 487                               : "+m" (*val), "=&r" (ret)
 488                               : "r" (new_val)
 489                               : "1", "cc");
 490
 491         return(ret);
 492 }
 493 # else
 494 static inline gpointer
 495 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 496 {
 497         gpointer ret;
 498
 499         __asm__ __volatile__ ("\tLA\t1,%0\n"
 500                               "0:\tLG\t%1,%0\n"
 501                               "\tCSG\t%1,%2,0(1)\n"
 502                               "\tJNZ\t0b"
 503                               : "+m" (*val), "=&r" (ret)
 504                               : "r" (new_val)
 505                               : "1", "cc");
 506
 507         return(ret);
 508 }
 509 # endif
 510
 511 # ifndef __s390x__
 512 static inline gint32
 513 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 514 {
 515         gint32 ret;
 516
 517         __asm__ __volatile__ ("\tLA\t2,%1\n"
 518                               "0:\tL\t%0,%1\n"
 519                               "\tLR\t1,%0\n"
 520                               "\tAR\t1,%2\n"
 521                               "\tCS\t%0,1,0(2)\n"
 522                               "\tJNZ\t0b"
 523                               : "=&r" (ret), "+m" (*val)
 524                               : "r" (add)
 525                               : "1", "2", "cc");
 526
 527         return(ret);
 528 }
 529 # else
 530 static inline gint32
 531 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 532 {
 533         gint32 ret;
 534
 535         __asm__ __volatile__ ("\tLA\t2,%1\n"
 536                               "0:\tLGF\t%0,%1\n"
 537                               "\tLGFR\t1,%0\n"
 538                               "\tAGR\t1,%2\n"
 539                               "\tCS\t%0,1,0(2)\n"
 540                               "\tJNZ\t0b"
 541                               : "=&r" (ret), "+m" (*val)
 542                               : "r" (add)
 543                               : "1", "2", "cc");
 544
 545         return(ret);
 546 }
 547 # endif
 548
 549 #elif defined(__mono_ppc__)
 550
 551 #ifdef G_COMPILER_CODEWARRIOR
 552 static inline gint32 InterlockedIncrement(volatile register gint32 *val)
 553 {
 554         gint32 result = 0, tmp;
 555         register gint32 result = 0;
 556         register gint32 tmp;
 557
 558         asm
 559         {
 560                 @1:
 561                         lwarx   tmp, 0, val
 562                         addi    result, tmp, 1
 563                         stwcx.  result, 0, val
 564                         bne-    @1
 565         }
 566
 567         return result;
 568 }
 569
 570 static inline gint32 InterlockedDecrement(register volatile gint32 *val)
 571 {
 572         register gint32 result = 0;
 573         register gint32 tmp;
 574
 575         asm
 576         {
 577                 @1:
 578                         lwarx   tmp, 0, val
 579                         addi    result, tmp, -1
 580                         stwcx.  result, 0, val
 581                         bne-    @1
 582         }
 583
 584         return result;
 585 }
 586 #define InterlockedCompareExchangePointer(dest,exch,comp) (void*)InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp))
 587
 588 static inline gint32 InterlockedCompareExchange(volatile register gint32 *dest, register gint32 exch, register gint32 comp)
 589 {
 590         register gint32 tmp = 0;
 591
 592         asm
 593         {
 594                 @1:
 595                         lwarx   tmp, 0, dest
 596                         cmpw    tmp, comp
 597                         bne-    @2
 598                         stwcx.  exch, 0, dest
 599                         bne-    @1
 600                 @2:
 601         }
 602
 603         return tmp;
 604 }
 605 static inline gint32 InterlockedExchange(register volatile gint32 *dest, register gint32 exch)
 606 {
 607         register gint32 tmp = 0;
 608
 609         asm
 610         {
 611                 @1:
 612                         lwarx   tmp, 0, dest
 613                         stwcx.  exch, 0, dest
 614                         bne-    @1
 615         }
 616
 617         return tmp;
 618 }
 619 #define InterlockedExchangePointer(dest,exch) (void*)InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch))
 620 #else
 621
 622 #if defined(__mono_ppc64__) && !defined(__mono_ilp32__)
 623 #define LDREGX "ldarx"
 624 #define STREGCXD "stdcx."
 625 #define CMPREG "cmpd"
 626 #else
 627 #define LDREGX "lwarx"
 628 #define STREGCXD "stwcx."
 629 #define CMPREG "cmpw"
 630 #endif
 631
 632 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 633 {
 634         gint32 result = 0, tmp;
 635
 636         __asm__ __volatile__ ("\n1:\n\t"
 637                               "lwarx  %0, 0, %2\n\t"
 638                               "addi   %1, %0, 1\n\t"
 639                               "stwcx. %1, 0, %2\n\t"
 640                               "bne-   1b"
 641                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 642         return result + 1;
 643 }
 644
 645 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 646 {
 647         gint32 result = 0, tmp;
 648
 649         __asm__ __volatile__ ("\n1:\n\t"
 650                               "lwarx  %0, 0, %2\n\t"
 651                               "addi   %1, %0, -1\n\t"
 652                               "stwcx. %1, 0, %2\n\t"
 653                               "bne-   1b"
 654                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 655         return result - 1;
 656 }
 657
 658 static inline gpointer InterlockedCompareExchangePointer (volatile gpointer *dest,
 659                                                 gpointer exch, gpointer comp)
 660 {
 661         gpointer tmp = NULL;
 662
 663         __asm__ __volatile__ ("\n1:\n\t"
 664                              LDREGX " %0, 0, %1\n\t"
 665                              CMPREG " %0, %2\n\t"
 666                              "bne-    2f\n\t"
 667                              STREGCXD " %3, 0, %1\n\t"
 668                              "bne-    1b\n"
 669                              "2:"
 670                              : "=&r" (tmp)
 671                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 672         return(tmp);
 673 }
 674
 675 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
 676                                                 gint32 exch, gint32 comp) {
 677         gint32 tmp = 0;
 678
 679         __asm__ __volatile__ ("\n1:\n\t"
 680                              "lwarx   %0, 0, %1\n\t"
 681                              "cmpw    %0, %2\n\t"
 682                              "bne-    2f\n\t"
 683                              "stwcx.  %3, 0, %1\n\t"
 684                              "bne-    1b\n"
 685                              "2:"
 686                              : "=&r" (tmp)
 687                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 688         return(tmp);
 689 }
 690
 691 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 692 {
 693         gint32 tmp = 0;
 694
 695         __asm__ __volatile__ ("\n1:\n\t"
 696                               "lwarx  %0, 0, %2\n\t"
 697                               "stwcx. %3, 0, %2\n\t"
 698                               "bne    1b"
 699                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 700         return(tmp);
 701 }
 702
 703 static inline gpointer InterlockedExchangePointer (volatile gpointer *dest, gpointer exch)
 704 {
 705         gpointer tmp = NULL;
 706
 707         __asm__ __volatile__ ("\n1:\n\t"
 708                               LDREGX " %0, 0, %2\n\t"
 709                               STREGCXD " %3, 0, %2\n\t"
 710                               "bne    1b"
 711                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 712         return(tmp);
 713 }
 714
 715 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 716 {
 717         gint32 result, tmp;
 718         __asm__ __volatile__ ("\n1:\n\t"
 719                               "lwarx  %0, 0, %2\n\t"
 720                               "add    %1, %0, %3\n\t"
 721                               "stwcx. %1, 0, %2\n\t"
 722                               "bne    1b"
 723                               : "=&r" (result), "=&r" (tmp)
 724                               : "r" (dest), "r" (add) : "cc", "memory");
 725         return(result);
 726 }
 727
 728 #undef LDREGX
 729 #undef STREGCXD
 730 #undef CMPREG
 731
 732 #endif /* !G_COMPILER_CODEWARRIOR */
 733
 734 #elif defined(__arm__)
 735
 736 /*
 737  * Atomic operations on ARM doesn't contain memory barriers, and the runtime code
 738  * depends on this, so we add them explicitly.
 739  */
 740
 741 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp)
 742 {
 743 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 744         gint32 ret, tmp;
 745         __asm__ __volatile__ (  "1:\n"
 746                                 "dmb\n"
 747                                 "mov    %0, #0\n"
 748                                 "ldrex %1, [%2]\n"
 749                                 "teq    %1, %3\n"
 750                                 "it eq\n"
 751                                 "strexeq %0, %4, [%2]\n"
 752                                 "teq %0, #0\n"
 753                                 "bne 1b\n"
 754                                 "dmb\n"
 755                                 : "=&r" (tmp), "=&r" (ret)
 756                                 : "r" (dest), "r" (comp), "r" (exch)
 757                                 : "memory", "cc");
 758
 759         return ret;
 760 #else
 761         gint32 a, b;
 762
 763         __asm__ __volatile__ (    "0:\n\t"
 764                                   "ldr %1, [%2]\n\t"
 765                                   "cmp %1, %4\n\t"
 766                                   "mov %0, %1\n\t"
 767                                   "bne 1f\n\t"
 768                                   "swp %0, %3, [%2]\n\t"
 769                                   "cmp %0, %1\n\t"
 770                                   "swpne %3, %0, [%2]\n\t"
 771                                   "bne 0b\n\t"
 772                                   "1:"
 773                                   : "=&r" (a), "=&r" (b)
 774                                   : "r" (dest), "r" (exch), "r" (comp)
 775                                   : "cc", "memory");
 776
 777         return a;
 778 #endif
 779 }
 780
 781 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
 782 {
 783 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 784         gpointer ret, tmp;
 785         __asm__ __volatile__ (
 786                                 "dmb\n"
 787                                 "1:\n"
 788                                 "mov    %0, #0\n"
 789                                 "ldrex %1, [%2]\n"
 790                                 "teq    %1, %3\n"
 791                                 "it eq\n"
 792                                 "strexeq %0, %4, [%2]\n"
 793                                 "teq %0, #0\n"
 794                                 "bne 1b\n"
 795                                 "dmb\n"
 796                                 : "=&r" (tmp), "=&r" (ret)
 797                                 : "r" (dest), "r" (comp), "r" (exch)
 798                                 : "memory", "cc");
 799
 800         return ret;
 801 #else
 802         gpointer a, b;
 803
 804         __asm__ __volatile__ (    "0:\n\t"
 805                                   "ldr %1, [%2]\n\t"
 806                                   "cmp %1, %4\n\t"
 807                                   "mov %0, %1\n\t"
 808                                   "bne 1f\n\t"
 809                                   "swpeq %0, %3, [%2]\n\t"
 810                                   "cmp %0, %1\n\t"
 811                                   "swpne %3, %0, [%2]\n\t"
 812                                   "bne 0b\n\t"
 813                                   "1:"
 814                                   : "=&r" (a), "=&r" (b)
 815                                   : "r" (dest), "r" (exch), "r" (comp)
 816                                   : "cc", "memory");
 817
 818         return a;
 819 #endif
 820 }
 821
 822 static inline gint32 InterlockedIncrement(volatile gint32 *dest)
 823 {
 824 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 825         gint32 ret, flag;
 826         __asm__ __volatile__ (
 827                                 "dmb\n"
 828                                 "1:\n"
 829                                 "ldrex %0, [%2]\n"
 830                                 "add %0, %0, %3\n"
 831                                 "strex %1, %0, [%2]\n"
 832                                 "teq %1, #0\n"
 833                                 "bne 1b\n"
 834                                 "dmb\n"
 835                                 : "=&r" (ret), "=&r" (flag)
 836                                 : "r" (dest), "r" (1)
 837                                 : "memory", "cc");
 838
 839         return ret;
 840 #else
 841         gint32 a, b, c;
 842
 843         __asm__ __volatile__ (  "0:\n\t"
 844                                 "ldr %0, [%3]\n\t"
 845                                 "add %1, %0, %4\n\t"
 846                                 "swp %2, %1, [%3]\n\t"
 847                                 "cmp %0, %2\n\t"
 848                                 "swpne %1, %2, [%3]\n\t"
 849                                 "bne 0b"
 850                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 851                                 : "r" (dest), "r" (1)
 852                                 : "cc", "memory");
 853
 854         return b;
 855 #endif
 856 }
 857
 858 static inline gint32 InterlockedDecrement(volatile gint32 *dest)
 859 {
 860 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 861         gint32 ret, flag;
 862         __asm__ __volatile__ (
 863                                 "dmb\n"
 864                                 "1:\n"
 865                                 "ldrex %0, [%2]\n"
 866                                 "sub %0, %0, %3\n"
 867                                 "strex %1, %0, [%2]\n"
 868                                 "teq %1, #0\n"
 869                                 "bne 1b\n"
 870                                 "dmb\n"
 871                                 : "=&r" (ret), "=&r" (flag)
 872                                 : "r" (dest), "r" (1)
 873                                 : "memory", "cc");
 874
 875         return ret;
 876 #else
 877         gint32 a, b, c;
 878
 879         __asm__ __volatile__ (  "0:\n\t"
 880                                 "ldr %0, [%3]\n\t"
 881                                 "add %1, %0, %4\n\t"
 882                                 "swp %2, %1, [%3]\n\t"
 883                                 "cmp %0, %2\n\t"
 884                                 "swpne %1, %2, [%3]\n\t"
 885                                 "bne 0b"
 886                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 887                                 : "r" (dest), "r" (-1)
 888                                 : "cc", "memory");
 889
 890         return b;
 891 #endif
 892 }
 893
 894 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 895 {
 896 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 897         gint32 ret, flag;
 898         __asm__ __volatile__ (
 899                                   "dmb\n"
 900                               "1:\n"
 901                               "ldrex %0, [%3]\n"
 902                               "strex %1, %2, [%3]\n"
 903                               "teq %1, #0\n"
 904                               "bne 1b\n"
 905                                   "dmb\n"
 906                               : "=&r" (ret), "=&r" (flag)
 907                               : "r" (exch), "r" (dest)
 908                               : "memory", "cc");
 909         return ret;
 910 #else
 911         gint32 a;
 912
 913         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 914                                 : "=&r" (a)
 915                                 : "r" (dest), "r" (exch));
 916
 917         return a;
 918 #endif
 919 }
 920
 921 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 922 {
 923 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 924         gpointer ret, flag;
 925         __asm__ __volatile__ (
 926                                   "dmb\n"
 927                               "1:\n"
 928                               "ldrex %0, [%3]\n"
 929                               "strex %1, %2, [%3]\n"
 930                               "teq %1, #0\n"
 931                               "bne 1b\n"
 932                                   "dmb\n"
 933                               : "=&r" (ret), "=&r" (flag)
 934                               : "r" (exch), "r" (dest)
 935                               : "memory", "cc");
 936         return ret;
 937 #else
 938         gpointer a;
 939
 940         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 941                                 : "=&r" (a)
 942                                 : "r" (dest), "r" (exch));
 943
 944         return a;
 945 #endif
 946 }
 947
 948 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 949 {
 950 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7S__)
 951         gint32 ret, tmp, flag;
 952         __asm__ __volatile__ (
 953                                 "dmb\n"
 954                                 "1:\n"
 955                                 "ldrex %0, [%3]\n"
 956                                 "add %1, %0, %4\n"
 957                                 "strex %2, %1, [%3]\n"
 958                                 "teq %2, #0\n"
 959                                 "bne 1b\n"
 960                                 "dmb\n"
 961                                 : "=&r" (ret), "=&r" (tmp), "=&r" (flag)
 962                                 : "r" (dest), "r" (add)
 963                                 : "memory", "cc");
 964
 965         return ret;
 966 #else
 967         int a, b, c;
 968
 969         __asm__ __volatile__ (  "0:\n\t"
 970                                 "ldr %0, [%3]\n\t"
 971                                 "add %1, %0, %4\n\t"
 972                                 "swp %2, %1, [%3]\n\t"
 973                                 "cmp %0, %2\n\t"
 974                                 "swpne %1, %2, [%3]\n\t"
 975                                 "bne 0b"
 976                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 977                                 : "r" (dest), "r" (add)
 978                                 : "cc", "memory");
 979
 980         return a;
 981 #endif
 982 }
 983
 984 #elif defined(__ia64__)
 985
 986 #ifdef __INTEL_COMPILER
 987 #include <ia64intrin.h>
 988 #endif
 989
 990 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
 991                                                 gint32 exch, gint32 comp)
 992 {
 993         gint32 old;
 994         guint64 real_comp;
 995
 996 #ifdef __INTEL_COMPILER
 997         old = _InterlockedCompareExchange (dest, exch, comp);
 998 #else
 999         /* cmpxchg4 zero extends the value read from memory */
1000         real_comp = (guint64)(guint32)comp;
1001         asm volatile ("mov ar.ccv = %2 ;;\n\t"
1002                                   "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
1003                                   : "=r" (old) : "r" (dest), "r" (real_comp), "r" (exch));
1004 #endif
1005
1006         return(old);
1007 }
1008
1009 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
1010                                                 gpointer exch, gpointer comp)
1011 {
1012         gpointer old;
1013
1014 #ifdef __INTEL_COMPILER
1015         old = _InterlockedCompareExchangePointer (dest, exch, comp);
1016 #else
1017         asm volatile ("mov ar.ccv = %2 ;;\n\t"
1018                                   "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
1019                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
1020 #endif
1021
1022         return(old);
1023 }
1024
1025 static inline gint32 InterlockedIncrement(gint32 volatile *val)
1026 {
1027 #ifdef __INTEL_COMPILER
1028         return _InterlockedIncrement (val);
1029 #else
1030         gint32 old;
1031
1032         do {
1033                 old = *val;
1034         } while (InterlockedCompareExchange (val, old + 1, old) != old);
1035
1036         return old + 1;
1037 #endif
1038 }
1039
1040 static inline gint32 InterlockedDecrement(gint32 volatile *val)
1041 {
1042 #ifdef __INTEL_COMPILER
1043         return _InterlockedDecrement (val);
1044 #else
1045         gint32 old;
1046
1047         do {
1048                 old = *val;
1049         } while (InterlockedCompareExchange (val, old - 1, old) != old);
1050
1051         return old - 1;
1052 #endif
1053 }
1054
1055 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
1056 {
1057 #ifdef __INTEL_COMPILER
1058         return _InterlockedExchange (dest, new_val);
1059 #else
1060         gint32 res;
1061
1062         do {
1063                 res = *dest;
1064         } while (InterlockedCompareExchange (dest, new_val, res) != res);
1065
1066         return res;
1067 #endif
1068 }
1069
1070 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
1071 {
1072 #ifdef __INTEL_COMPILER
1073         return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
1074 #else
1075         gpointer res;
1076
1077         do {
1078                 res = *dest;
1079         } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
1080
1081         return res;
1082 #endif
1083 }
1084
1085 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
1086 {
1087         gint32 old;
1088
1089 #ifdef __INTEL_COMPILER
1090         old = _InterlockedExchangeAdd (val, add);
1091 #else
1092         do {
1093                 old = *val;
1094         } while (InterlockedCompareExchange (val, old + add, old) != old);
1095
1096         return old;
1097 #endif
1098 }
1099
1100 #elif defined(__mips__)
1101
1102 #if SIZEOF_REGISTER == 8
1103 #error "Not implemented."
1104 #endif
1105
1106 static inline gint32 InterlockedIncrement(volatile gint32 *val)
1107 {
1108         gint32 tmp, result = 0;
1109
1110         __asm__ __volatile__ ("    .set    mips32\n"
1111                               "1:  ll      %0, %2\n"
1112                               "    addu    %1, %0, 1\n"
1113                               "    sc      %1, %2\n"
1114                               "    beqz    %1, 1b\n"
1115                               "    .set    mips0\n"
1116                               : "=&r" (result), "=&r" (tmp), "=m" (*val)
1117                               : "m" (*val));
1118         return result + 1;
1119 }
1120
1121 static inline gint32 InterlockedDecrement(volatile gint32 *val)
1122 {
1123         gint32 tmp, result = 0;
1124
1125         __asm__ __volatile__ ("    .set    mips32\n"
1126                               "1:  ll      %0, %2\n"
1127                               "    subu    %1, %0, 1\n"
1128                               "    sc      %1, %2\n"
1129                               "    beqz    %1, 1b\n"
1130                               "    .set    mips0\n"
1131                               : "=&r" (result), "=&r" (tmp), "=m" (*val)
1132                               : "m" (*val));
1133         return result - 1;
1134 }
1135
1136 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
1137                                                 gint32 exch, gint32 comp) {
1138         gint32 old, tmp;
1139
1140         __asm__ __volatile__ ("    .set    mips32\n"
1141                               "1:  ll      %0, %2\n"
1142                               "    bne     %0, %5, 2f\n"
1143                               "    move    %1, %4\n"
1144                               "    sc      %1, %2\n"
1145                               "    beqz    %1, 1b\n"
1146                               "2:  .set    mips0\n"
1147                               : "=&r" (old), "=&r" (tmp), "=m" (*dest)
1148                               : "m" (*dest), "r" (exch), "r" (comp));
1149         return(old);
1150 }
1151
1152 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
1153 {
1154         return (gpointer)(InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp)));
1155 }
1156
1157 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
1158 {
1159         gint32 result, tmp;
1160
1161         __asm__ __volatile__ ("    .set    mips32\n"
1162                               "1:  ll      %0, %2\n"
1163                               "    move    %1, %4\n"
1164                               "    sc      %1, %2\n"
1165                               "    beqz    %1, 1b\n"
1166                               "    .set    mips0\n"
1167                               : "=&r" (result), "=&r" (tmp), "=m" (*dest)
1168                               : "m" (*dest), "r" (exch));
1169         return(result);
1170 }
1171
1172 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
1173 {
1174         return (gpointer)InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch));
1175 }
1176
1177 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
1178 {
1179         gint32 result, tmp;
1180
1181         __asm__ __volatile__ ("    .set    mips32\n"
1182                               "1:  ll      %0, %2\n"
1183                               "    addu    %1, %0, %4\n"
1184                               "    sc      %1, %2\n"
1185                               "    beqz    %1, 1b\n"
1186                               "    .set    mips0\n"
1187                               : "=&r" (result), "=&r" (tmp), "=m" (*dest)
1188                               : "m" (*dest), "r" (add));
1189         return result;
1190 }
1191
1192 #else
1193
1194 #define WAPI_NO_ATOMIC_ASM
1195
1196 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
1197 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
1198 extern gint32 InterlockedIncrement(volatile gint32 *dest);
1199 extern gint32 InterlockedDecrement(volatile gint32 *dest);
1200 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
1201 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
1202 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
1203
1204 #endif
1205
1206 /* Not yet used */
1207 #ifdef USE_GCC_ATOMIC_OPS
1208
1209 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
1210                                                 gint32 exch, gint32 comp)
1211 {
1212         return __sync_val_compare_and_swap (dest, comp, exch);
1213 }
1214
1215 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
1216 {
1217         return __sync_val_compare_and_swap (dest, comp, exch);
1218 }
1219
1220 static inline gint32 InterlockedIncrement(volatile gint32 *val)
1221 {
1222         return __sync_add_and_fetch (val, 1);
1223 }
1224
1225 static inline gint32 InterlockedDecrement(volatile gint32 *val)
1226 {
1227         return __sync_add_and_fetch (val, -1);
1228 }
1229
1230 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
1231 {
1232         gint32 old_val;
1233         do {
1234                 old_val = *val;
1235         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
1236         return old_val;
1237 }
1238
1239 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
1240                                                   gpointer new_val)
1241 {
1242         gpointer old_val;
1243         do {
1244                 old_val = *val;
1245         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
1246         return old_val;
1247 }
1248
1249 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
1250 {
1251         return __sync_fetch_and_add (val, add);
1252 }
1253 #endif
1254
1255 #endif /* _WAPI_ATOMIC_H_ */