mono/io-layer/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  */
   9
  10 #ifndef _WAPI_ATOMIC_H_
  11 #define _WAPI_ATOMIC_H_
  12
  13 #if defined(__NetBSD__)
  14 #include <sys/param.h>
  15
  16 #if __NetBSD_Version__ > 499004000
  17 #include <sys/atomic.h>
  18 #define HAVE_ATOMIC_OPS
  19 #endif
  20
  21 #endif
  22
  23 #include <glib.h>
  24
  25 #include "mono/io-layer/wapi.h"
  26
  27 #if defined(__NetBSD__) && defined(HAVE_ATOMIC_OPS)
  28
  29 #define WAPI_ATOMIC_ASM
  30 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  31        gint32 exch, gint32 comp)
  32 {
  33        return atomic_cas_32((uint32_t*)dest, comp, exch);
  34 }
  35
  36 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  37 {
  38        return atomic_cas_ptr(dest, comp, exch);
  39 }
  40
  41 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  42 {
  43        return atomic_inc_32_nv((uint32_t*)val);
  44 }
  45
  46 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  47 {
  48        return atomic_dec_32_nv((uint32_t*)val);
  49 }
  50
  51 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  52 {
  53        return atomic_swap_32((uint32_t*)val, new_val);
  54 }
  55
  56 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
  57                gpointer new_val)
  58 {
  59        return atomic_swap_ptr(val, new_val);
  60 }
  61
  62 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
  63 {
  64        return atomic_add_32_nv((uint32_t*)val, add) - add;
  65 }
  66
  67 #elif defined(__i386__) || defined(__x86_64__)
  68 #define WAPI_ATOMIC_ASM
  69
  70 /*
  71  * NB: The *Pointer() functions here assume that
  72  * sizeof(pointer)==sizeof(gint32)
  73  *
  74  * NB2: These asm functions assume 486+ (some of the opcodes dont
  75  * exist on 386).  If this becomes an issue, we can get configure to
  76  * fall back to the non-atomic C versions of these calls.
  77  */
  78
  79 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  80                                                 gint32 exch, gint32 comp)
  81 {
  82         gint32 old;
  83
  84         __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  85                               : "=m" (*dest), "=a" (old)
  86                               : "r" (exch), "m" (*dest), "a" (comp));
  87         return(old);
  88 }
  89
  90 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  91 {
  92         gpointer old;
  93
  94         __asm__ __volatile__ ("lock; "
  95 #ifdef __x86_64__
  96                               "cmpxchgq"
  97 #else
  98                               "cmpxchgl"
  99 #endif
 100                               " %2, %0"
 101                               : "=m" (*dest), "=a" (old)
 102                               : "r" (exch), "m" (*dest), "a" (comp));
 103
 104         return(old);
 105 }
 106
 107 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 108 {
 109         gint32 tmp;
 110
 111         __asm__ __volatile__ ("lock; xaddl %0, %1"
 112                               : "=r" (tmp), "=m" (*val)
 113                               : "0" (1), "m" (*val));
 114
 115         return(tmp+1);
 116 }
 117
 118 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 119 {
 120         gint32 tmp;
 121
 122         __asm__ __volatile__ ("lock; xaddl %0, %1"
 123                               : "=r" (tmp), "=m" (*val)
 124                               : "0" (-1), "m" (*val));
 125
 126         return(tmp-1);
 127 }
 128
 129 /*
 130  * See
 131  * http://msdn.microsoft.com/library/en-us/dnmag00/html/win320700.asp?frame=true
 132  * for the reasons for using cmpxchg and a loop here.
 133  *
 134  * That url is no longer valid, but it's still in the google cache at the
 135  * moment: http://www.google.com/search?q=cache:http://msdn.microsoft.com/library/en-us/dnmag00/html/win320700.asp?frame=true
 136  *
 137  * For the time being, http://msdn.microsoft.com/msdnmag/issues/0700/Win32/
 138  * might work.  Bet it will change soon enough though.
 139  */
 140 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 141 {
 142         gint32 ret;
 143
 144         __asm__ __volatile__ ("1:; lock; cmpxchgl %2, %0; jne 1b"
 145                               : "=m" (*val), "=a" (ret)
 146                               : "r" (new_val), "m" (*val), "a" (*val));
 147
 148         return(ret);
 149 }
 150
 151 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
 152                                                   gpointer new_val)
 153 {
 154         gpointer ret;
 155
 156         __asm__ __volatile__ ("1:; lock; "
 157 #ifdef __x86_64__
 158                               "cmpxchgq"
 159 #else
 160                               "cmpxchgl"
 161 #endif
 162                               " %2, %0; jne 1b"
 163                               : "=m" (*val), "=a" (ret)
 164                               : "r" (new_val), "m" (*val), "a" (*val));
 165
 166         return(ret);
 167 }
 168
 169 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 170 {
 171         gint32 ret;
 172
 173         __asm__ __volatile__ ("lock; xaddl %0, %1"
 174                               : "=r" (ret), "=m" (*val)
 175                               : "0" (add), "m" (*val));
 176
 177         return(ret);
 178 }
 179
 180 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
 181 #define WAPI_ATOMIC_ASM
 182
 183 G_GNUC_UNUSED
 184 static inline gint32 InterlockedCompareExchange(volatile gint32 *_dest, gint32 _exch, gint32 _comp)
 185 {
 186        register volatile gint32 *dest asm("g1") = _dest;
 187        register gint32 comp asm("o4") = _comp;
 188        register gint32 exch asm("o5") = _exch;
 189
 190        __asm__ __volatile__(
 191                /* cas [%%g1], %%o4, %%o5 */
 192                ".word 0xdbe0500c"
 193                : "=r" (exch)
 194                : "0" (exch), "r" (dest), "r" (comp)
 195                : "memory");
 196
 197        return exch;
 198 }
 199
 200 G_GNUC_UNUSED
 201 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *_dest, gpointer _exch, gpointer _comp)
 202 {
 203        register volatile gpointer *dest asm("g1") = _dest;
 204        register gpointer comp asm("o4") = _comp;
 205        register gpointer exch asm("o5") = _exch;
 206
 207        __asm__ __volatile__(
 208 #ifdef SPARCV9
 209                /* casx [%%g1], %%o4, %%o5 */
 210                ".word 0xdbf0500c"
 211 #else
 212                /* cas [%%g1], %%o4, %%o5 */
 213                ".word 0xdbe0500c"
 214 #endif
 215                : "=r" (exch)
 216                : "0" (exch), "r" (dest), "r" (comp)
 217                : "memory");
 218
 219        return exch;
 220 }
 221
 222 G_GNUC_UNUSED
 223 static inline gint32 InterlockedIncrement(volatile gint32 *_dest)
 224 {
 225        register volatile gint32 *dest asm("g1") = _dest;
 226        register gint32 tmp asm("o4");
 227        register gint32 ret asm("o5");
 228
 229        __asm__ __volatile__(
 230                "1:     ld      [%%g1], %%o4\n\t"
 231                "       add     %%o4, 1, %%o5\n\t"
 232                /*      cas     [%%g1], %%o4, %%o5 */
 233                "       .word   0xdbe0500c\n\t"
 234                "       cmp     %%o4, %%o5\n\t"
 235                "       bne     1b\n\t"
 236                "        add    %%o5, 1, %%o5"
 237                : "=&r" (tmp), "=&r" (ret)
 238                : "r" (dest)
 239                : "memory", "cc");
 240
 241         return ret;
 242 }
 243
 244 G_GNUC_UNUSED
 245 static inline gint32 InterlockedDecrement(volatile gint32 *_dest)
 246 {
 247        register volatile gint32 *dest asm("g1") = _dest;
 248        register gint32 tmp asm("o4");
 249        register gint32 ret asm("o5");
 250
 251        __asm__ __volatile__(
 252                "1:     ld      [%%g1], %%o4\n\t"
 253                "       sub     %%o4, 1, %%o5\n\t"
 254                /*      cas     [%%g1], %%o4, %%o5 */
 255                "       .word   0xdbe0500c\n\t"
 256                "       cmp     %%o4, %%o5\n\t"
 257                "       bne     1b\n\t"
 258                "        sub    %%o5, 1, %%o5"
 259                : "=&r" (tmp), "=&r" (ret)
 260                : "r" (dest)
 261                : "memory", "cc");
 262
 263         return ret;
 264 }
 265
 266 G_GNUC_UNUSED
 267 static inline gint32 InterlockedExchange(volatile gint32 *_dest, gint32 exch)
 268 {
 269        register volatile gint32 *dest asm("g1") = _dest;
 270        register gint32 tmp asm("o4");
 271        register gint32 ret asm("o5");
 272
 273        __asm__ __volatile__(
 274                "1:     ld      [%%g1], %%o4\n\t"
 275                "       mov     %3, %%o5\n\t"
 276                /*      cas     [%%g1], %%o4, %%o5 */
 277                "       .word   0xdbe0500c\n\t"
 278                "       cmp     %%o4, %%o5\n\t"
 279                "       bne     1b\n\t"
 280                "        nop"
 281                : "=&r" (tmp), "=&r" (ret)
 282                : "r" (dest), "r" (exch)
 283                : "memory", "cc");
 284
 285         return ret;
 286 }
 287
 288 G_GNUC_UNUSED
 289 static inline gpointer InterlockedExchangePointer(volatile gpointer *_dest, gpointer exch)
 290 {
 291        register volatile gpointer *dest asm("g1") = _dest;
 292        register gpointer tmp asm("o4");
 293        register gpointer ret asm("o5");
 294
 295        __asm__ __volatile__(
 296 #ifdef SPARCV9
 297                "1:     ldx     [%%g1], %%o4\n\t"
 298 #else
 299                "1:     ld      [%%g1], %%o4\n\t"
 300 #endif
 301                "       mov     %3, %%o5\n\t"
 302 #ifdef SPARCV9
 303                /*      casx    [%%g1], %%o4, %%o5 */
 304                "       .word   0xdbf0500c\n\t"
 305 #else
 306                /*      cas     [%%g1], %%o4, %%o5 */
 307                "       .word   0xdbe0500c\n\t"
 308 #endif
 309                "       cmp     %%o4, %%o5\n\t"
 310                "       bne     1b\n\t"
 311                "        nop"
 312                : "=&r" (tmp), "=&r" (ret)
 313                : "r" (dest), "r" (exch)
 314                : "memory", "cc");
 315
 316         return ret;
 317 }
 318
 319 G_GNUC_UNUSED
 320 static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
 321 {
 322        register volatile gint32 *dest asm("g1") = _dest;
 323        register gint32 tmp asm("o4");
 324        register gint32 ret asm("o5");
 325
 326        __asm__ __volatile__(
 327                "1:     ld      [%%g1], %%o4\n\t"
 328                "       add     %%o4, %3, %%o5\n\t"
 329                /*      cas     [%%g1], %%o4, %%o5 */
 330                "       .word   0xdbe0500c\n\t"
 331                "       cmp     %%o4, %%o5\n\t"
 332                "       bne     1b\n\t"
 333                "        add    %%o5, %3, %%o5"
 334                : "=&r" (tmp), "=&r" (ret)
 335                : "r" (dest), "r" (add)
 336                : "memory", "cc");
 337
 338         return ret;
 339 }
 340
 341 #elif __s390__
 342
 343 #define WAPI_ATOMIC_ASM
 344
 345 static inline gint32
 346 InterlockedCompareExchange(volatile gint32 *dest,
 347                            gint32 exch, gint32 comp)
 348 {
 349         gint32 old;
 350
 351         __asm__ __volatile__ ("\tLA\t1,%0\n"
 352                               "\tLR\t%1,%3\n"
 353                               "\tCS\t%1,%2,0(1)\n"
 354                               : "+m" (*dest), "=&r" (old)
 355                               : "r" (exch), "r" (comp)
 356                               : "1", "cc");
 357         return(old);
 358 }
 359
 360 #ifndef __s390x__
 361 static inline gpointer
 362 InterlockedCompareExchangePointer(volatile gpointer *dest,
 363                            gpointer exch, gpointer comp)
 364 {
 365         gpointer old;
 366
 367         __asm__ __volatile__ ("\tLA\t1,%0\n"
 368                               "\tLR\t%1,%3\n"
 369                               "\tCS\t%1,%2,0(1)\n"
 370                               : "+m" (*dest), "=&r" (old)
 371                               : "r" (exch), "r" (comp)
 372                               : "1", "cc");
 373         return(old);
 374 }
 375 # else
 376 static inline gpointer
 377 InterlockedCompareExchangePointer(volatile gpointer *dest,
 378                                   gpointer exch,
 379                                   gpointer comp)
 380 {
 381         gpointer old;
 382
 383         __asm__ __volatile__ ("\tLA\t1,%0\n"
 384                               "\tLGR\t%1,%3\n"
 385                               "\tCSG\t%1,%2,0(1)\n"
 386                               : "+m" (*dest), "=&r" (old)
 387                               : "r" (exch), "r" (comp)
 388                               : "1", "cc");
 389
 390         return(old);
 391 }
 392 # endif
 393
 394 # ifndef __s390x__
 395 static inline gint32
 396 InterlockedIncrement(volatile gint32 *val)
 397 {
 398         gint32 tmp;
 399
 400         __asm__ __volatile__ ("\tLA\t2,%1\n"
 401                               "0:\tL\t%0,%1\n"
 402                               "\tLR\t1,%0\n"
 403                               "\tAHI\t1,1\n"
 404                               "\tCS\t%0,1,0(2)\n"
 405                               "\tJNZ\t0b\n"
 406                               "\tLR\t%0,1"
 407                               : "=r" (tmp), "+m" (*val)
 408                               : : "1", "2", "cc");
 409
 410         return(tmp);
 411 }
 412 # else
 413 static inline gint32
 414 InterlockedIncrement(volatile gint32 *val)
 415 {
 416         gint32 tmp;
 417
 418         __asm__ __volatile__ ("\tLA\t2,%1\n"
 419                               "0:\tLGF\t%0,%1\n"
 420                               "\tLGFR\t1,%0\n"
 421                               "\tAGHI\t1,1\n"
 422                               "\tCS\t%0,1,0(2)\n"
 423                               "\tJNZ\t0b\n"
 424                               "\tLGFR\t%0,1"
 425                               : "=r" (tmp), "+m" (*val)
 426                               : : "1", "2", "cc");
 427
 428         return(tmp);
 429 }
 430 # endif
 431
 432 # ifndef __s390x__
 433 static inline gint32
 434 InterlockedDecrement(volatile gint32 *val)
 435 {
 436         gint32 tmp;
 437
 438         __asm__ __volatile__ ("\tLA\t2,%1\n"
 439                               "0:\tL\t%0,%1\n"
 440                               "\tLR\t1,%0\n"
 441                               "\tAHI\t1,-1\n"
 442                               "\tCS\t%0,1,0(2)\n"
 443                               "\tJNZ\t0b\n"
 444                               "\tLR\t%0,1"
 445                               : "=r" (tmp), "+m" (*val)
 446                               : : "1", "2", "cc");
 447
 448         return(tmp);
 449 }
 450 # else
 451 static inline gint32
 452 InterlockedDecrement(volatile gint32 *val)
 453 {
 454         gint32 tmp;
 455
 456         __asm__ __volatile__ ("\tLA\t2,%1\n"
 457                               "0:\tLGF\t%0,%1\n"
 458                               "\tLGFR\t1,%0\n"
 459                               "\tAGHI\t1,-1\n"
 460                               "\tCS\t%0,1,0(2)\n"
 461                               "\tJNZ\t0b\n"
 462                               "\tLGFR\t%0,1"
 463                               : "=r" (tmp), "+m" (*val)
 464                               : : "1", "2", "cc");
 465
 466         return(tmp);
 467 }
 468 # endif
 469
 470 static inline gint32
 471 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 472 {
 473         gint32 ret;
 474
 475         __asm__ __volatile__ ("\tLA\t1,%0\n"
 476                               "0:\tL\t%1,%0\n"
 477                               "\tCS\t%1,%2,0(1)\n"
 478                               "\tJNZ\t0b"
 479                               : "+m" (*val), "=&r" (ret)
 480                               : "r" (new_val)
 481                               : "1", "cc");
 482
 483         return(ret);
 484 }
 485
 486 # ifndef __s390x__
 487 static inline gpointer
 488 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 489 {
 490         gpointer ret;
 491
 492         __asm__ __volatile__ ("\tLA\t1,%0\n"
 493                               "0:\tL\t%1,%0\n"
 494                               "\tCS\t%1,%2,0(1)\n"
 495                               "\tJNZ\t0b"
 496                               : "+m" (*val), "=&r" (ret)
 497                               : "r" (new_val)
 498                               : "1", "cc");
 499
 500         return(ret);
 501 }
 502 # else
 503 static inline gpointer
 504 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 505 {
 506         gpointer ret;
 507
 508         __asm__ __volatile__ ("\tLA\t1,%0\n"
 509                               "0:\tLG\t%1,%0\n"
 510                               "\tCSG\t%1,%2,0(1)\n"
 511                               "\tJNZ\t0b"
 512                               : "+m" (*val), "=&r" (ret)
 513                               : "r" (new_val)
 514                               : "1", "cc");
 515
 516         return(ret);
 517 }
 518 # endif
 519
 520 # ifndef __s390x__
 521 static inline gint32
 522 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 523 {
 524         gint32 ret;
 525
 526         __asm__ __volatile__ ("\tLA\t2,%1\n"
 527                               "0:\tL\t%0,%1\n"
 528                               "\tLR\t1,%0\n"
 529                               "\tAR\t1,%2\n"
 530                               "\tCS\t%0,1,0(2)\n"
 531                               "\tJNZ\t0b"
 532                               : "=&r" (ret), "+m" (*val)
 533                               : "r" (add)
 534                               : "1", "2", "cc");
 535
 536         return(ret);
 537 }
 538 # else
 539 static inline gint32
 540 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 541 {
 542         gint32 ret;
 543
 544         __asm__ __volatile__ ("\tLA\t2,%1\n"
 545                               "0:\tLGF\t%0,%1\n"
 546                               "\tLGFR\t1,%0\n"
 547                               "\tAGR\t1,%2\n"
 548                               "\tCS\t%0,1,0(2)\n"
 549                               "\tJNZ\t0b"
 550                               : "=&r" (ret), "+m" (*val)
 551                               : "r" (add)
 552                               : "1", "2", "cc");
 553
 554         return(ret);
 555 }
 556 # endif
 557
 558 #elif defined(__mono_ppc__)
 559 #define WAPI_ATOMIC_ASM
 560
 561 #ifdef G_COMPILER_CODEWARRIOR
 562 static inline gint32 InterlockedIncrement(volatile register gint32 *val)
 563 {
 564         gint32 result = 0, tmp;
 565         register gint32 result = 0;
 566         register gint32 tmp;
 567
 568         asm
 569         {
 570                 @1:
 571                         lwarx   tmp, 0, val
 572                         addi    result, tmp, 1
 573                         stwcx.  result, 0, val
 574                         bne-    @1
 575         }
 576
 577         return result;
 578 }
 579
 580 static inline gint32 InterlockedDecrement(register volatile gint32 *val)
 581 {
 582         register gint32 result = 0;
 583         register gint32 tmp;
 584
 585         asm
 586         {
 587                 @1:
 588                         lwarx   tmp, 0, val
 589                         addi    result, tmp, -1
 590                         stwcx.  result, 0, val
 591                         bne-    @1
 592         }
 593
 594         return result;
 595 }
 596 #define InterlockedCompareExchangePointer(dest,exch,comp) (void*)InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp))
 597
 598 static inline gint32 InterlockedCompareExchange(volatile register gint32 *dest, register gint32 exch, register gint32 comp)
 599 {
 600         register gint32 tmp = 0;
 601
 602         asm
 603         {
 604                 @1:
 605                         lwarx   tmp, 0, dest
 606                         cmpw    tmp, comp
 607                         bne-    @2
 608                         stwcx.  exch, 0, dest
 609                         bne-    @1
 610                 @2:
 611         }
 612
 613         return tmp;
 614 }
 615 static inline gint32 InterlockedExchange(register volatile gint32 *dest, register gint32 exch)
 616 {
 617         register gint32 tmp = 0;
 618
 619         asm
 620         {
 621                 @1:
 622                         lwarx   tmp, 0, dest
 623                         stwcx.  exch, 0, dest
 624                         bne-    @1
 625         }
 626
 627         return tmp;
 628 }
 629 #define InterlockedExchangePointer(dest,exch) (void*)InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch))
 630 #else
 631
 632 #if defined(__mono_ppc64__) && !defined(__mono_ilp32__)
 633 #define LDREGX "ldarx"
 634 #define STREGCXD "stdcx."
 635 #define CMPREG "cmpd"
 636 #else
 637 #define LDREGX "lwarx"
 638 #define STREGCXD "stwcx."
 639 #define CMPREG "cmpw"
 640 #endif
 641
 642 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 643 {
 644         gint32 result = 0, tmp;
 645
 646         __asm__ __volatile__ ("\n1:\n\t"
 647                               "lwarx  %0, 0, %2\n\t"
 648                               "addi   %1, %0, 1\n\t"
 649                               "stwcx. %1, 0, %2\n\t"
 650                               "bne-   1b"
 651                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 652         return result + 1;
 653 }
 654
 655 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 656 {
 657         gint32 result = 0, tmp;
 658
 659         __asm__ __volatile__ ("\n1:\n\t"
 660                               "lwarx  %0, 0, %2\n\t"
 661                               "addi   %1, %0, -1\n\t"
 662                               "stwcx. %1, 0, %2\n\t"
 663                               "bne-   1b"
 664                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 665         return result - 1;
 666 }
 667
 668 static inline gpointer InterlockedCompareExchangePointer (volatile gpointer *dest,
 669                                                 gpointer exch, gpointer comp)
 670 {
 671         gpointer tmp = NULL;
 672
 673         __asm__ __volatile__ ("\n1:\n\t"
 674                              LDREGX " %0, 0, %1\n\t"
 675                              CMPREG " %0, %2\n\t"
 676                              "bne-    2f\n\t"
 677                              STREGCXD " %3, 0, %1\n\t"
 678                              "bne-    1b\n"
 679                              "2:"
 680                              : "=&r" (tmp)
 681                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 682         return(tmp);
 683 }
 684
 685 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
 686                                                 gint32 exch, gint32 comp) {
 687         gint32 tmp = 0;
 688
 689         __asm__ __volatile__ ("\n1:\n\t"
 690                              "lwarx   %0, 0, %1\n\t"
 691                              "cmpw    %0, %2\n\t"
 692                              "bne-    2f\n\t"
 693                              "stwcx.  %3, 0, %1\n\t"
 694                              "bne-    1b\n"
 695                              "2:"
 696                              : "=&r" (tmp)
 697                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 698         return(tmp);
 699 }
 700
 701 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 702 {
 703         gint32 tmp = 0;
 704
 705         __asm__ __volatile__ ("\n1:\n\t"
 706                               "lwarx  %0, 0, %2\n\t"
 707                               "stwcx. %3, 0, %2\n\t"
 708                               "bne    1b"
 709                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 710         return(tmp);
 711 }
 712
 713 static inline gpointer InterlockedExchangePointer (volatile gpointer *dest, gpointer exch)
 714 {
 715         gpointer tmp = NULL;
 716
 717         __asm__ __volatile__ ("\n1:\n\t"
 718                               LDREGX " %0, 0, %2\n\t"
 719                               STREGCXD " %3, 0, %2\n\t"
 720                               "bne    1b"
 721                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 722         return(tmp);
 723 }
 724
 725 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 726 {
 727         gint32 result, tmp;
 728         __asm__ __volatile__ ("\n1:\n\t"
 729                               "lwarx  %0, 0, %2\n\t"
 730                               "add    %1, %0, %3\n\t"
 731                               "stwcx. %1, 0, %2\n\t"
 732                               "bne    1b"
 733                               : "=&r" (result), "=&r" (tmp)
 734                               : "r" (dest), "r" (add) : "cc", "memory");
 735         return(result);
 736 }
 737
 738 #undef LDREGX
 739 #undef STREGCXD
 740 #undef CMPREG
 741
 742 #endif /* !G_COMPILER_CODEWARRIOR */
 743
 744 #elif defined(__arm__)
 745 #define WAPI_ATOMIC_ASM
 746
 747 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp)
 748 {
 749         int a, b;
 750
 751         __asm__ __volatile__ (    "0:\n\t"
 752                                   "ldr %1, [%2]\n\t"
 753                                   "cmp %1, %4\n\t"
 754                                   "mov %0, %1\n\t"
 755                                   "bne 1f\n\t"
 756                                   "swp %0, %3, [%2]\n\t"
 757                                   "cmp %0, %1\n\t"
 758                                   "swpne %3, %0, [%2]\n\t"
 759                                   "bne 0b\n\t"
 760                                   "1:"
 761                                   : "=&r" (a), "=&r" (b)
 762                                   : "r" (dest), "r" (exch), "r" (comp)
 763                                   : "cc", "memory");
 764
 765         return a;
 766 }
 767
 768 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
 769 {
 770         gpointer a, b;
 771
 772         __asm__ __volatile__ (    "0:\n\t"
 773                                   "ldr %1, [%2]\n\t"
 774                                   "cmp %1, %4\n\t"
 775                                   "mov %0, %1\n\t"
 776                                   "bne 1f\n\t"
 777                                   "swpeq %0, %3, [%2]\n\t"
 778                                   "cmp %0, %1\n\t"
 779                                   "swpne %3, %0, [%2]\n\t"
 780                                   "bne 0b\n\t"
 781                                   "1:"
 782                                   : "=&r" (a), "=&r" (b)
 783                                   : "r" (dest), "r" (exch), "r" (comp)
 784                                   : "cc", "memory");
 785
 786         return a;
 787 }
 788
 789 static inline gint32 InterlockedIncrement(volatile gint32 *dest)
 790 {
 791         int a, b, c;
 792
 793         __asm__ __volatile__ (  "0:\n\t"
 794                                 "ldr %0, [%3]\n\t"
 795                                 "add %1, %0, %4\n\t"
 796                                 "swp %2, %1, [%3]\n\t"
 797                                 "cmp %0, %2\n\t"
 798                                 "swpne %1, %2, [%3]\n\t"
 799                                 "bne 0b"
 800                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 801                                 : "r" (dest), "r" (1)
 802                                 : "cc", "memory");
 803
 804         return b;
 805 }
 806
 807 static inline gint32 InterlockedDecrement(volatile gint32 *dest)
 808 {
 809         int a, b, c;
 810
 811         __asm__ __volatile__ (  "0:\n\t"
 812                                 "ldr %0, [%3]\n\t"
 813                                 "add %1, %0, %4\n\t"
 814                                 "swp %2, %1, [%3]\n\t"
 815                                 "cmp %0, %2\n\t"
 816                                 "swpne %1, %2, [%3]\n\t"
 817                                 "bne 0b"
 818                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 819                                 : "r" (dest), "r" (-1)
 820                                 : "cc", "memory");
 821
 822         return b;
 823 }
 824
 825 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 826 {
 827         int a;
 828
 829         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 830                                 : "=&r" (a)
 831                                 : "r" (dest), "r" (exch));
 832
 833         return a;
 834 }
 835
 836 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 837 {
 838         gpointer a;
 839
 840         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 841                                 : "=&r" (a)
 842                                 : "r" (dest), "r" (exch));
 843
 844         return a;
 845 }
 846
 847 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 848 {
 849         int a, b, c;
 850
 851         __asm__ __volatile__ (  "0:\n\t"
 852                                 "ldr %0, [%3]\n\t"
 853                                 "add %1, %0, %4\n\t"
 854                                 "swp %2, %1, [%3]\n\t"
 855                                 "cmp %0, %2\n\t"
 856                                 "swpne %1, %2, [%3]\n\t"
 857                                 "bne 0b"
 858                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 859                                 : "r" (dest), "r" (add)
 860                                 : "cc", "memory");
 861
 862         return a;
 863 }
 864
 865 #elif defined(__ia64__)
 866 #define WAPI_ATOMIC_ASM
 867
 868 #ifdef __INTEL_COMPILER
 869 #include <ia64intrin.h>
 870 #endif
 871
 872 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
 873                                                 gint32 exch, gint32 comp)
 874 {
 875         gint32 old;
 876         guint64 real_comp;
 877
 878 #ifdef __INTEL_COMPILER
 879         old = _InterlockedCompareExchange (dest, exch, comp);
 880 #else
 881         /* cmpxchg4 zero extends the value read from memory */
 882         real_comp = (guint64)(guint32)comp;
 883         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 884                                   "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
 885                                   : "=r" (old) : "r" (dest), "r" (real_comp), "r" (exch));
 886 #endif
 887
 888         return(old);
 889 }
 890
 891 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
 892                                                 gpointer exch, gpointer comp)
 893 {
 894         gpointer old;
 895
 896 #ifdef __INTEL_COMPILER
 897         old = _InterlockedCompareExchangePointer (dest, exch, comp);
 898 #else
 899         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 900                                   "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
 901                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
 902 #endif
 903
 904         return(old);
 905 }
 906
 907 static inline gint32 InterlockedIncrement(gint32 volatile *val)
 908 {
 909 #ifdef __INTEL_COMPILER
 910         return _InterlockedIncrement (val);
 911 #else
 912         gint32 old;
 913
 914         do {
 915                 old = *val;
 916         } while (InterlockedCompareExchange (val, old + 1, old) != old);
 917
 918         return old + 1;
 919 #endif
 920 }
 921
 922 static inline gint32 InterlockedDecrement(gint32 volatile *val)
 923 {
 924 #ifdef __INTEL_COMPILER
 925         return _InterlockedDecrement (val);
 926 #else
 927         gint32 old;
 928
 929         do {
 930                 old = *val;
 931         } while (InterlockedCompareExchange (val, old - 1, old) != old);
 932
 933         return old - 1;
 934 #endif
 935 }
 936
 937 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
 938 {
 939 #ifdef __INTEL_COMPILER
 940         return _InterlockedExchange (dest, new_val);
 941 #else
 942         gint32 res;
 943
 944         do {
 945                 res = *dest;
 946         } while (InterlockedCompareExchange (dest, new_val, res) != res);
 947
 948         return res;
 949 #endif
 950 }
 951
 952 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
 953 {
 954 #ifdef __INTEL_COMPILER
 955         return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
 956 #else
 957         gpointer res;
 958
 959         do {
 960                 res = *dest;
 961         } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
 962
 963         return res;
 964 #endif
 965 }
 966
 967 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
 968 {
 969         gint32 old;
 970
 971 #ifdef __INTEL_COMPILER
 972         old = _InterlockedExchangeAdd (val, add);
 973 #else
 974         do {
 975                 old = *val;
 976         } while (InterlockedCompareExchange (val, old + add, old) != old);
 977
 978         return old;
 979 #endif
 980 }
 981
 982 #elif defined(__alpha__)
 983 #define WAPI_ATOMIC_ASM
 984
 985 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
 986                                                 gint32 exch, gint32 comp)
 987 {
 988         gint32 old, temp, temp2;
 989         long compq = comp, exchq = exch;
 990
 991         __asm__ __volatile__ (
 992                 "1:     ldl_l %2, %0\n"
 993                 "       mov %2, %1\n"
 994                 "       cmpeq %2, %5, %3\n"
 995                 "       cmovne %3, %4, %2\n"
 996                 "       stl_c %2, %0\n"
 997                 "       beq %2, 1b\n"
 998                 : "=m" (*dest), "=&r" (old), "=&r" (temp), "=&r" (temp2)
 999                 : "r" (exchq), "r" (compq), "m" (*dest));
1000         return(old);
1001 }
1002
1003 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
1004 {
1005         gpointer old, temp, temp2;
1006
1007         __asm__ __volatile__ (
1008                 "1:     ldq_l %2, %0\n"
1009                 "       mov %2, %1\n"
1010                 "       cmpeq %2, %5, %3\n"
1011                 "       cmovne %3, %4, %2\n"
1012                 "       stq_c %2, %0\n"
1013                 "       beq %2, 1b\n"
1014                 : "=m" (*dest), "=&r" (old), "=&r" (temp), "=&r" (temp2)
1015                 : "r" (exch), "r" (comp), "m" (*dest));
1016         return(old);
1017 }
1018
1019 static inline gint32 InterlockedIncrement(volatile gint32 *val)
1020 {
1021         gint32 temp, cur;
1022
1023         __asm__ __volatile__ (
1024                 "1:     ldl_l %0, %1\n"
1025                 "       addl %0, %3, %0\n"
1026                 "       mov %0, %2\n"
1027                 "       stl_c %0, %1\n"
1028                 "       beq %0, 1b\n"
1029                 : "=&r" (temp), "=m" (*val), "=r" (cur)
1030                 : "Ir" (1), "m" (*val));
1031         return(cur);
1032 }
1033
1034 static inline gint32 InterlockedDecrement(volatile gint32 *val)
1035 {
1036         gint32 temp, cur;
1037
1038         __asm__ __volatile__ (
1039                 "1:     ldl_l %0, %1\n"
1040                 "       subl %0, %3, %0\n"
1041                 "       mov %0, %2\n"
1042                 "       stl_c %0, %1\n"
1043                 "       beq %0, 1b\n"
1044                 : "=&r" (temp), "=m" (*val), "=r" (cur)
1045                 : "Ir" (1), "m" (*val));
1046         return(cur);
1047 }
1048
1049 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
1050 {
1051         gint32 ret, temp;
1052
1053         __asm__ __volatile__ (
1054                 "1:     ldl_l %1, %0\n"
1055                 "       mov %3, %2\n"
1056                 "       stl_c %2, %0\n"
1057                 "       beq %2, 1b\n"
1058                 : "=m" (*val), "=&r" (ret), "=&r" (temp)
1059                 : "r" (new_val), "m" (*val));
1060         return(ret);
1061 }
1062
1063 static inline gpointer InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
1064 {
1065         gpointer ret, temp;
1066
1067         __asm__ __volatile__ (
1068                 "1:     ldq_l %1, %0\n"
1069                 "       mov %3, %2\n"
1070                 "       stq_c %2, %0\n"
1071                 "       beq %2, 1b\n"
1072                 : "=m" (*val), "=&r" (ret), "=&r" (temp)
1073                 : "r" (new_val), "m" (*val));
1074         return(ret);
1075 }
1076
1077 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
1078 {
1079         gint32 ret, temp;
1080
1081         __asm__ __volatile__ (
1082                 "1:     ldl_l   %2, %0\n"
1083                 "       mov     %2, %1\n"
1084                 "       addl    %2, %3, %2\n"
1085                 "       stl_c   %2, %0\n"
1086                 "       beq     %2, 1b\n"
1087                 : "=m" (*val), "=&r" (ret), "=&r" (temp)
1088                 : "r" (add), "m" (*val));
1089
1090         return(ret);
1091 }
1092
1093 #elif defined(__mips__)
1094 #define WAPI_ATOMIC_ASM
1095
1096 static inline gint32 InterlockedIncrement(volatile gint32 *val)
1097 {
1098         gint32 tmp, result = 0;
1099
1100         __asm__ __volatile__ ("    .set    mips32\n"
1101                               "1:  ll      %0, %2\n"
1102                               "    addu    %1, %0, 1\n"
1103                               "    sc      %1, %2\n"
1104                               "    beqz    %1, 1b\n"
1105                               "    .set    mips0\n"
1106                               : "=&r" (result), "=&r" (tmp), "=m" (*val)
1107                               : "m" (*val));
1108         return result + 1;
1109 }
1110
1111 static inline gint32 InterlockedDecrement(volatile gint32 *val)
1112 {
1113         gint32 tmp, result = 0;
1114
1115         __asm__ __volatile__ ("    .set    mips32\n"
1116                               "1:  ll      %0, %2\n"
1117                               "    subu    %1, %0, 1\n"
1118                               "    sc      %1, %2\n"
1119                               "    beqz    %1, 1b\n"
1120                               "    .set    mips0\n"
1121                               : "=&r" (result), "=&r" (tmp), "=m" (*val)
1122                               : "m" (*val));
1123         return result - 1;
1124 }
1125
1126 #define InterlockedCompareExchangePointer(dest,exch,comp) InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp))
1127
1128 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
1129                                                 gint32 exch, gint32 comp) {
1130         gint32 old, tmp;
1131
1132         __asm__ __volatile__ ("    .set    mips32\n"
1133                               "1:  ll      %0, %2\n"
1134                               "    bne     %0, %5, 2f\n"
1135                               "    move    %1, %4\n"
1136                               "    sc      %1, %2\n"
1137                               "    beqz    %1, 1b\n"
1138                               "2:  .set    mips0\n"
1139                               : "=&r" (old), "=&r" (tmp), "=m" (*dest)
1140                               : "m" (*dest), "r" (exch), "r" (comp));
1141         return(old);
1142 }
1143
1144 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
1145 {
1146         gint32 result, tmp;
1147
1148         __asm__ __volatile__ ("    .set    mips32\n"
1149                               "1:  ll      %0, %2\n"
1150                               "    move    %1, %4\n"
1151                               "    sc      %1, %2\n"
1152                               "    beqz    %1, 1b\n"
1153                               "    .set    mips0\n"
1154                               : "=&r" (result), "=&r" (tmp), "=m" (*dest)
1155                               : "m" (*dest), "r" (exch));
1156         return(result);
1157 }
1158 #define InterlockedExchangePointer(dest,exch) InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch))
1159
1160 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
1161 {
1162         gint32 result, tmp;
1163
1164         __asm__ __volatile__ ("    .set    mips32\n"
1165                               "1:  ll      %0, %2\n"
1166                               "    addu    %1, %0, %4\n"
1167                               "    sc      %1, %2\n"
1168                               "    beqz    %1, 1b\n"
1169                               "    .set    mips0\n"
1170                               : "=&r" (result), "=&r" (tmp), "=m" (*dest)
1171                               : "m" (*dest), "r" (add));
1172         return result;
1173 }
1174
1175 #else
1176
1177 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
1178 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
1179 extern gint32 InterlockedIncrement(volatile gint32 *dest);
1180 extern gint32 InterlockedDecrement(volatile gint32 *dest);
1181 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
1182 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
1183 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
1184
1185 #if defined(__hppa__)
1186 #define WAPI_ATOMIC_ASM
1187 #endif
1188
1189 #endif
1190
1191 #endif /* _WAPI_ATOMIC_H_ */