mono/io-layer/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  */
   9
  10 #ifndef _WAPI_ATOMIC_H_
  11 #define _WAPI_ATOMIC_H_
  12
  13 #include <glib.h>
  14
  15 #include "mono/io-layer/wapi.h"
  16
  17 #if defined(__i386__) || defined(__x86_64__)
  18 #define WAPI_ATOMIC_ASM
  19
  20 /*
  21  * NB: The *Pointer() functions here assume that
  22  * sizeof(pointer)==sizeof(gint32)
  23  *
  24  * NB2: These asm functions assume 486+ (some of the opcodes dont
  25  * exist on 386).  If this becomes an issue, we can get configure to
  26  * fall back to the non-atomic C versions of these calls.
  27  */
  28
  29 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  30                                                 gint32 exch, gint32 comp)
  31 {
  32         gint32 old;
  33
  34         __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  35                               : "=m" (*dest), "=a" (old)
  36                               : "r" (exch), "m" (*dest), "a" (comp));
  37         return(old);
  38 }
  39
  40 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  41 {
  42         gpointer old;
  43
  44         __asm__ __volatile__ ("lock; "
  45 #ifdef __x86_64__
  46                               "cmpxchgq"
  47 #else
  48                               "cmpxchgl"
  49 #endif
  50                               " %2, %0"
  51                               : "=m" (*dest), "=a" (old)
  52                               : "r" (exch), "m" (*dest), "a" (comp));
  53
  54         return(old);
  55 }
  56
  57 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  58 {
  59         gint32 tmp;
  60
  61         __asm__ __volatile__ ("lock; xaddl %0, %1"
  62                               : "=r" (tmp), "=m" (*val)
  63                               : "0" (1), "m" (*val));
  64
  65         return(tmp+1);
  66 }
  67
  68 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  69 {
  70         gint32 tmp;
  71
  72         __asm__ __volatile__ ("lock; xaddl %0, %1"
  73                               : "=r" (tmp), "=m" (*val)
  74                               : "0" (-1), "m" (*val));
  75
  76         return(tmp-1);
  77 }
  78
  79 /*
  80  * See
  81  * http://msdn.microsoft.com/library/en-us/dnmag00/html/win320700.asp?frame=true
  82  * for the reasons for using cmpxchg and a loop here.
  83  *
  84  * That url is no longer valid, but it's still in the google cache at the
  85  * moment: http://www.google.com/search?q=cache:http://msdn.microsoft.com/library/en-us/dnmag00/html/win320700.asp?frame=true
  86  *
  87  * For the time being, http://msdn.microsoft.com/msdnmag/issues/0700/Win32/
  88  * might work.  Bet it will change soon enough though.
  89  */
  90 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  91 {
  92         gint32 ret;
  93
  94         __asm__ __volatile__ ("1:; lock; cmpxchgl %2, %0; jne 1b"
  95                               : "=m" (*val), "=a" (ret)
  96                               : "r" (new_val), "m" (*val), "a" (*val));
  97
  98         return(ret);
  99 }
 100
 101 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
 102                                                   gpointer new_val)
 103 {
 104         gpointer ret;
 105
 106         __asm__ __volatile__ ("1:; lock; "
 107 #ifdef __x86_64__
 108                               "cmpxchgq"
 109 #else
 110                               "cmpxchgl"
 111 #endif
 112                               " %2, %0; jne 1b"
 113                               : "=m" (*val), "=a" (ret)
 114                               : "r" (new_val), "m" (*val), "a" (*val));
 115
 116         return(ret);
 117 }
 118
 119 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 120 {
 121         gint32 ret;
 122
 123         __asm__ __volatile__ ("lock; xaddl %0, %1"
 124                               : "=r" (ret), "=m" (*val)
 125                               : "0" (add), "m" (*val));
 126
 127         return(ret);
 128 }
 129
 130 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
 131 #define WAPI_ATOMIC_ASM
 132
 133 G_GNUC_UNUSED
 134 static inline gint32 InterlockedCompareExchange(volatile gint32 *_dest, gint32 _exch, gint32 _comp)
 135 {
 136        register volatile gint32 *dest asm("g1") = _dest;
 137        register gint32 comp asm("o4") = _comp;
 138        register gint32 exch asm("o5") = _exch;
 139
 140        __asm__ __volatile__(
 141                /* cas [%%g1], %%o4, %%o5 */
 142                ".word 0xdbe0500c"
 143                : "=r" (exch)
 144                : "0" (exch), "r" (dest), "r" (comp)
 145                : "memory");
 146
 147        return exch;
 148 }
 149
 150 G_GNUC_UNUSED
 151 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *_dest, gpointer _exch, gpointer _comp)
 152 {
 153        register volatile gpointer *dest asm("g1") = _dest;
 154        register gpointer comp asm("o4") = _comp;
 155        register gpointer exch asm("o5") = _exch;
 156
 157        __asm__ __volatile__(
 158 #ifdef SPARCV9
 159                /* casx [%%g1], %%o4, %%o5 */
 160                ".word 0xdbf0500c"
 161 #else
 162                /* cas [%%g1], %%o4, %%o5 */
 163                ".word 0xdbe0500c"
 164 #endif
 165                : "=r" (exch)
 166                : "0" (exch), "r" (dest), "r" (comp)
 167                : "memory");
 168
 169        return exch;
 170 }
 171
 172 G_GNUC_UNUSED
 173 static inline gint32 InterlockedIncrement(volatile gint32 *_dest)
 174 {
 175        register volatile gint32 *dest asm("g1") = _dest;
 176        register gint32 tmp asm("o4");
 177        register gint32 ret asm("o5");
 178
 179        __asm__ __volatile__(
 180                "1:     ld      [%%g1], %%o4\n\t"
 181                "       add     %%o4, 1, %%o5\n\t"
 182                /*      cas     [%%g1], %%o4, %%o5 */
 183                "       .word   0xdbe0500c\n\t"
 184                "       cmp     %%o4, %%o5\n\t"
 185                "       bne     1b\n\t"
 186                "        add    %%o5, 1, %%o5"
 187                : "=&r" (tmp), "=&r" (ret)
 188                : "r" (dest)
 189                : "memory", "cc");
 190
 191         return ret;
 192 }
 193
 194 G_GNUC_UNUSED
 195 static inline gint32 InterlockedDecrement(volatile gint32 *_dest)
 196 {
 197        register volatile gint32 *dest asm("g1") = _dest;
 198        register gint32 tmp asm("o4");
 199        register gint32 ret asm("o5");
 200
 201        __asm__ __volatile__(
 202                "1:     ld      [%%g1], %%o4\n\t"
 203                "       sub     %%o4, 1, %%o5\n\t"
 204                /*      cas     [%%g1], %%o4, %%o5 */
 205                "       .word   0xdbe0500c\n\t"
 206                "       cmp     %%o4, %%o5\n\t"
 207                "       bne     1b\n\t"
 208                "        sub    %%o5, 1, %%o5"
 209                : "=&r" (tmp), "=&r" (ret)
 210                : "r" (dest)
 211                : "memory", "cc");
 212
 213         return ret;
 214 }
 215
 216 G_GNUC_UNUSED
 217 static inline gint32 InterlockedExchange(volatile gint32 *_dest, gint32 exch)
 218 {
 219        register volatile gint32 *dest asm("g1") = _dest;
 220        register gint32 tmp asm("o4");
 221        register gint32 ret asm("o5");
 222
 223        __asm__ __volatile__(
 224                "1:     ld      [%%g1], %%o4\n\t"
 225                "       mov     %3, %%o5\n\t"
 226                /*      cas     [%%g1], %%o4, %%o5 */
 227                "       .word   0xdbe0500c\n\t"
 228                "       cmp     %%o4, %%o5\n\t"
 229                "       bne     1b\n\t"
 230                "        nop"
 231                : "=&r" (tmp), "=&r" (ret)
 232                : "r" (dest), "r" (exch)
 233                : "memory", "cc");
 234
 235         return ret;
 236 }
 237
 238 G_GNUC_UNUSED
 239 static inline gpointer InterlockedExchangePointer(volatile gpointer *_dest, gpointer exch)
 240 {
 241        register volatile gpointer *dest asm("g1") = _dest;
 242        register gpointer tmp asm("o4");
 243        register gpointer ret asm("o5");
 244
 245        __asm__ __volatile__(
 246 #ifdef SPARCV9
 247                "1:     ldx     [%%g1], %%o4\n\t"
 248 #else
 249                "1:     ld      [%%g1], %%o4\n\t"
 250 #endif
 251                "       mov     %3, %%o5\n\t"
 252 #ifdef SPARCV9
 253                /*      casx    [%%g1], %%o4, %%o5 */
 254                "       .word   0xdbf0500c\n\t"
 255 #else
 256                /*      cas     [%%g1], %%o4, %%o5 */
 257                "       .word   0xdbe0500c\n\t"
 258 #endif
 259                "       cmp     %%o4, %%o5\n\t"
 260                "       bne     1b\n\t"
 261                "        nop"
 262                : "=&r" (tmp), "=&r" (ret)
 263                : "r" (dest), "r" (exch)
 264                : "memory", "cc");
 265
 266         return ret;
 267 }
 268
 269 G_GNUC_UNUSED
 270 static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
 271 {
 272        register volatile gint32 *dest asm("g1") = _dest;
 273        register gint32 tmp asm("o4");
 274        register gint32 ret asm("o5");
 275
 276        __asm__ __volatile__(
 277                "1:     ld      [%%g1], %%o4\n\t"
 278                "       add     %%o4, %3, %%o5\n\t"
 279                /*      cas     [%%g1], %%o4, %%o5 */
 280                "       .word   0xdbe0500c\n\t"
 281                "       cmp     %%o4, %%o5\n\t"
 282                "       bne     1b\n\t"
 283                "        add    %%o5, %3, %%o5"
 284                : "=&r" (tmp), "=&r" (ret)
 285                : "r" (dest), "r" (add)
 286                : "memory", "cc");
 287
 288         return ret;
 289 }
 290
 291 #elif __s390__
 292
 293 #define WAPI_ATOMIC_ASM
 294
 295 static inline gint32
 296 InterlockedCompareExchange(volatile gint32 *dest,
 297                            gint32 exch, gint32 comp)
 298 {
 299         gint32 old;
 300
 301         __asm__ __volatile__ ("\tLA\t1,%0\n"
 302                               "0:\tL\t%1,%0\n"
 303                               "\tCR\t%1,%3\n"
 304                               "\tJNE\t1f\n"
 305                               "\tCS\t%1,%2,0(1)\n"
 306                               "\tJNZ\t0b\n"
 307                               "1:\n"
 308                               : "+m" (*dest), "=r" (old)
 309                               : "r" (exch), "r" (comp)
 310                               : "1", "cc");
 311         return(old);
 312 }
 313
 314 #ifndef __s390x__
 315 static inline gpointer
 316 InterlockedCompareExchangePointer(volatile gpointer *dest,
 317                            gpointer exch, gpointer comp)
 318 {
 319         gpointer old;
 320
 321         __asm__ __volatile__ ("\tLA\t1,%0\n"
 322                               "0:\tL\t%1,%0\n"
 323                               "\tCR\t%1,%3\n"
 324                               "\tJNE\t1f\n"
 325                               "\tCS\t%1,%2,0(1)\n"
 326                               "\tJNZ\t0b\n"
 327                               "1:\n"
 328                               : "+m" (*dest), "=r" (old)
 329                               : "r" (exch), "r" (comp)
 330                               : "1", "cc");
 331         return(old);
 332 }
 333 # else
 334 static inline gpointer
 335 InterlockedCompareExchangePointer(volatile gpointer *dest,
 336                                   gpointer exch,
 337                                   gpointer comp)
 338 {
 339         gpointer old;
 340
 341         __asm__ __volatile__ ("\tLA\t1,%0\n"
 342                               "0:\tLG\t%1,%0\n"
 343                               "\tCGR\t%1,%3\n"
 344                               "\tJNE\t1f\n"
 345                               "\tCSG\t%1,%2,0(1)\n"
 346                               "\tJNZ\t0b\n"
 347                               "1:\n"
 348                               : "+m" (*dest), "=r" (old)
 349                               : "r" (exch), "r" (comp)
 350                               : "1", "cc");
 351
 352         return(old);
 353 }
 354 # endif
 355
 356 # ifndef __s390x__
 357 static inline gint32
 358 InterlockedIncrement(volatile gint32 *val)
 359 {
 360         gint32 tmp;
 361
 362         __asm__ __volatile__ ("\tLA\t2,%1\n"
 363                               "0:\tL\t%0,%1\n"
 364                               "\tLR\t1,%0\n"
 365                               "\tAHI\t1,1\n"
 366                               "\tCS\t%0,1,0(2)\n"
 367                               "\tJNZ\t0b\n"
 368                               "\tLR\t%0,1"
 369                               : "=r" (tmp), "+m" (*val)
 370                               : : "1", "2", "cc");
 371
 372         return(tmp);
 373 }
 374 # else
 375 static inline gint32
 376 InterlockedIncrement(volatile gint32 *val)
 377 {
 378         gint32 tmp;
 379
 380         __asm__ __volatile__ ("\tLA\t2,%1\n"
 381                               "0:\tLGF\t%0,%1\n"
 382                               "\tLGFR\t1,%0\n"
 383                               "\tAGHI\t1,1\n"
 384                               "\tCS\t%0,1,0(2)\n"
 385                               "\tJNZ\t0b\n"
 386                               "\tLGFR\t%0,1"
 387                               : "=r" (tmp), "+m" (*val)
 388                               : : "1", "2", "cc");
 389
 390         return(tmp);
 391 }
 392 # endif
 393
 394 # ifndef __s390x__
 395 static inline gint32
 396 InterlockedDecrement(volatile gint32 *val)
 397 {
 398         gint32 tmp;
 399
 400         __asm__ __volatile__ ("\tLA\t2,%1\n"
 401                               "0:\tL\t%0,%1\n"
 402                               "\tLR\t1,%0\n"
 403                               "\tAHI\t1,-1\n"
 404                               "\tCS\t%0,1,0(2)\n"
 405                               "\tJNZ\t0b\n"
 406                               "\tLR\t%0,1"
 407                               : "=r" (tmp), "+m" (*val)
 408                               : : "1", "2", "cc");
 409
 410         return(tmp);
 411 }
 412 # else
 413 static inline gint32
 414 InterlockedDecrement(volatile gint32 *val)
 415 {
 416         gint32 tmp;
 417
 418         __asm__ __volatile__ ("\tLA\t2,%1\n"
 419                               "0:\tLGF\t%0,%1\n"
 420                               "\tLGFR\t1,%0\n"
 421                               "\tAGHI\t1,-1\n"
 422                               "\tCS\t%0,1,0(2)\n"
 423                               "\tJNZ\t0b\n"
 424                               "\tLGFR\t%0,1"
 425                               : "=r" (tmp), "+m" (*val)
 426                               : : "1", "2", "cc");
 427
 428         return(tmp);
 429 }
 430 # endif
 431
 432 static inline gint32
 433 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 434 {
 435         gint32 ret;
 436
 437         __asm__ __volatile__ ("\tLA\t1,%0\n"
 438                               "0:\tL\t%1,%0\n"
 439                               "\tCS\t%1,%2,0(1)\n"
 440                               "\tJNZ\t0b"
 441                               : "+m" (*val), "=r" (ret)
 442                               : "r" (new_val)
 443                               : "1", "cc");
 444
 445         return(ret);
 446 }
 447
 448 # ifndef __s390x__
 449 static inline gpointer
 450 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 451 {
 452         gpointer ret;
 453
 454         __asm__ __volatile__ ("\tLA\t1,%0\n"
 455                               "0:\tL\t%1,%0\n"
 456                               "\tCS\t%1,%2,0(1)\n"
 457                               "\tJNZ\t0b"
 458                               : "+m" (*val), "=r" (ret)
 459                               : "r" (new_val)
 460                               : "1", "cc");
 461
 462         return(ret);
 463 }
 464 # else
 465 static inline gpointer
 466 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 467 {
 468         gpointer ret;
 469
 470         __asm__ __volatile__ ("\tLA\t1,%0\n"
 471                               "0:\tLG\t%1,%0\n"
 472                               "\tCSG\t%1,%2,0(1)\n"
 473                               "\tJNZ\t0b"
 474                               : "+m" (*val), "=r" (ret)
 475                               : "r" (new_val)
 476                               : "1", "cc");
 477
 478         return(ret);
 479 }
 480 # endif
 481
 482 # ifndef __s390x__
 483 static inline gint32
 484 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 485 {
 486         gint32 ret;
 487
 488         __asm__ __volatile__ ("\tLA\t2,%1\n"
 489                               "0:\tL\t%0,%1\n"
 490                               "\tLR\t1,%0\n"
 491                               "\tAR\t1,%2\n"
 492                               "\tCS\t%0,1,0(2)\n"
 493                               "\tJNZ\t0b"
 494                               : "=r" (ret), "+m" (*val)
 495                               : "r" (add)
 496                               : "1", "2", "cc");
 497
 498         return(ret);
 499 }
 500 # else
 501 static inline gint32
 502 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 503 {
 504         gint32 ret;
 505
 506         __asm__ __volatile__ ("\tLA\t2,%1\n"
 507                               "0:\tLGF\t%0,%1\n"
 508                               "\tLGFR\t1,%0\n"
 509                               "\tAGR\t1,%2\n"
 510                               "\tCS\t%0,1,0(2)\n"
 511                               "\tJNZ\t0b"
 512                               : "=r" (ret), "+m" (*val)
 513                               : "r" (add)
 514                               : "1", "2", "cc");
 515
 516         return(ret);
 517 }
 518 # endif
 519
 520 #elif defined(__ppc__) || defined (__powerpc__)
 521 #define WAPI_ATOMIC_ASM
 522
 523 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 524 {
 525         gint32 result = 0, tmp;
 526
 527         __asm__ __volatile__ ("\n1:\n\t"
 528                               "lwarx  %0, 0, %2\n\t"
 529                               "addi   %1, %0, 1\n\t"
 530                               "stwcx. %1, 0, %2\n\t"
 531                               "bne-   1b"
 532                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 533         return result + 1;
 534 }
 535
 536 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 537 {
 538         gint32 result = 0, tmp;
 539
 540         __asm__ __volatile__ ("\n1:\n\t"
 541                               "lwarx  %0, 0, %2\n\t"
 542                               "addi   %1, %0, -1\n\t"
 543                               "stwcx. %1, 0, %2\n\t"
 544                               "bne-   1b"
 545                               : "=&b" (result), "=&b" (tmp): "r" (val): "cc", "memory");
 546         return result - 1;
 547 }
 548
 549 #define InterlockedCompareExchangePointer(dest,exch,comp) InterlockedCompareExchange((volatile gint32 *)(dest), (gint32)(exch), (gint32)(comp))
 550
 551 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
 552                                                 gint32 exch, gint32 comp) {
 553         gint32 tmp = 0;
 554
 555         __asm__ __volatile__ ("\n1:\n\t"
 556                              "lwarx   %0, 0, %1\n\t"
 557                              "cmpw    %0, %2\n\t"
 558                              "bne-    2f\n\t"
 559                              "stwcx.  %3, 0, %1\n\t"
 560                              "bne-    1b\n"
 561                              "2:"
 562                              : "=&r" (tmp)
 563                              : "b" (dest), "r" (comp), "r" (exch): "cc", "memory");
 564         return(tmp);
 565 }
 566
 567 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 568 {
 569         gint32 tmp = 0;
 570
 571         __asm__ __volatile__ ("\n1:\n\t"
 572                               "lwarx  %0, 0, %2\n\t"
 573                               "stwcx. %3, 0, %2\n\t"
 574                               "bne    1b"
 575                               : "=r" (tmp) : "0" (tmp), "b" (dest), "r" (exch): "cc", "memory");
 576         return(tmp);
 577 }
 578 #define InterlockedExchangePointer(dest,exch) InterlockedExchange((volatile gint32 *)(dest), (gint32)(exch))
 579
 580 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 581 {
 582         gint32 result, tmp;
 583         __asm__ __volatile__ ("\n1:\n\t"
 584                               "lwarx  %0, 0, %2\n\t"
 585                               "add    %1, %0, %3\n\t"
 586                               "stwcx. %1, 0, %2\n\t"
 587                               "bne    1b"
 588                               : "=&r" (result), "=&r" (tmp)
 589                               : "r" (dest), "r" (add) : "cc", "memory");
 590         return(result);
 591 }
 592
 593 #elif defined(__arm__)
 594 #define WAPI_ATOMIC_ASM
 595
 596 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp)
 597 {
 598         int a, b;
 599
 600         __asm__ __volatile__ (    "0:\n\t"
 601                                   "ldr %1, [%2]\n\t"
 602                                   "cmp %1, %4\n\t"
 603                                   "bne 1f\n\t"
 604                                   "swp %0, %3, [%2]\n\t"
 605                                   "cmp %0, %1\n\t"
 606                                   "swpne %3, %0, [%2]\n\t"
 607                                   "bne 0b\n\t"
 608                                   "1:"
 609                                   : "=&r" (a), "=&r" (b)
 610                                   : "r" (dest), "r" (exch), "r" (comp)
 611                                   : "cc", "memory");
 612
 613         return a;
 614 }
 615
 616 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
 617 {
 618         gpointer a, b;
 619
 620         __asm__ __volatile__ (    "0:\n\t"
 621                                   "ldr %1, [%2]\n\t"
 622                                   "cmp %1, %4\n\t"
 623                                   "bne 1f\n\t"
 624                                   "swpeq %0, %3, [%2]\n\t"
 625                                   "cmp %0, %1\n\t"
 626                                   "swpne %3, %0, [%2]\n\t"
 627                                   "bne 0b\n\t"
 628                                   "1:"
 629                                   : "=&r" (a), "=&r" (b)
 630                                   : "r" (dest), "r" (exch), "r" (comp)
 631                                   : "cc", "memory");
 632
 633         return a;
 634 }
 635
 636 static inline gint32 InterlockedIncrement(volatile gint32 *dest)
 637 {
 638         int a, b, c;
 639
 640         __asm__ __volatile__ (  "0:\n\t"
 641                                 "ldr %0, [%3]\n\t"
 642                                 "add %1, %0, %4\n\t"
 643                                 "swp %2, %1, [%3]\n\t"
 644                                 "cmp %0, %2\n\t"
 645                                 "swpne %1, %2, [%3]\n\t"
 646                                 "bne 0b"
 647                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 648                                 : "r" (dest), "r" (1)
 649                                 : "cc", "memory");
 650
 651         return b;
 652 }
 653
 654 static inline gint32 InterlockedDecrement(volatile gint32 *dest)
 655 {
 656         int a, b, c;
 657
 658         __asm__ __volatile__ (  "0:\n\t"
 659                                 "ldr %0, [%3]\n\t"
 660                                 "add %1, %0, %4\n\t"
 661                                 "swp %2, %1, [%3]\n\t"
 662                                 "cmp %0, %2\n\t"
 663                                 "swpne %1, %2, [%3]\n\t"
 664                                 "bne 0b"
 665                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 666                                 : "r" (dest), "r" (-1)
 667                                 : "cc", "memory");
 668
 669         return b;
 670 }
 671
 672 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 673 {
 674         int a;
 675
 676         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 677                                 : "=&r" (a)
 678                                 : "r" (dest), "r" (exch));
 679
 680         return a;
 681 }
 682
 683 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 684 {
 685         gpointer a;
 686
 687         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 688                                 : "=&r" (a)
 689                                 : "r" (dest), "r" (exch));
 690
 691         return a;
 692 }
 693
 694 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 695 {
 696         int a, b, c;
 697
 698         __asm__ __volatile__ (  "0:\n\t"
 699                                 "ldr %0, [%3]\n\t"
 700                                 "add %1, %0, %4\n\t"
 701                                 "swp %2, %1, [%3]\n\t"
 702                                 "cmp %0, %2\n\t"
 703                                 "swpne %1, %2, [%3]\n\t"
 704                                 "bne 0b"
 705                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 706                                 : "r" (dest), "r" (add)
 707                                 : "cc", "memory");
 708
 709         return a;
 710 }
 711
 712 #elif defined(__ia64__)
 713 #define WAPI_ATOMIC_ASM
 714
 715 #ifdef __INTEL_COMPILER
 716 #include <ia64intrin.h>
 717 #endif
 718
 719 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
 720                                                 gint32 exch, gint32 comp)
 721 {
 722         gint32 old;
 723
 724 #ifdef __INTEL_COMPILER
 725         old = _InterlockedCompareExchange (dest, exch, comp);
 726 #else
 727         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 728                                   "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
 729                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
 730 #endif
 731
 732         return(old);
 733 }
 734
 735 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
 736                                                 gpointer exch, gpointer comp)
 737 {
 738         gpointer old;
 739
 740 #ifdef __INTEL_COMPILER
 741         old = _InterlockedCompareExchangePointer (dest, exch, comp);
 742 #else
 743         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 744                                   "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
 745                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
 746 #endif
 747
 748         return(old);
 749 }
 750
 751 static inline gint32 InterlockedIncrement(gint32 volatile *val)
 752 {
 753 #ifdef __INTEL_COMPILER
 754         return _InterlockedIncrement (val);
 755 #else
 756         gint32 old;
 757
 758         do {
 759                 old = *val;
 760         } while (InterlockedCompareExchange (val, old + 1, old) != old);
 761
 762         return old + 1;
 763 #endif
 764 }
 765
 766 static inline gint32 InterlockedDecrement(gint32 volatile *val)
 767 {
 768 #ifdef __INTEL_COMPILER
 769         return _InterlockedDecrement (val);
 770 #else
 771         gint32 old;
 772
 773         do {
 774                 old = *val;
 775         } while (InterlockedCompareExchange (val, old - 1, old) != old);
 776
 777         return old - 1;
 778 #endif
 779 }
 780
 781 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
 782 {
 783 #ifdef __INTEL_COMPILER
 784         return _InterlockedExchange (dest, new_val);
 785 #else
 786         gint32 res;
 787
 788         do {
 789                 res = *dest;
 790         } while (InterlockedCompareExchange (dest, new_val, res) != res);
 791
 792         return res;
 793 #endif
 794 }
 795
 796 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
 797 {
 798 #ifdef __INTEL_COMPILER
 799         return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
 800 #else
 801         gpointer res;
 802
 803         do {
 804                 res = *dest;
 805         } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
 806
 807         return res;
 808 #endif
 809 }
 810
 811 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
 812 {
 813         gint32 old;
 814
 815 #ifdef __INTEL_COMPILER
 816         old = _InterlockedExchangeAdd (val, add);
 817 #else
 818         do {
 819                 old = *val;
 820         } while (InterlockedCompareExchange (val, old + add, old) != old);
 821
 822         return old;
 823 #endif
 824 }
 825
 826 #else
 827
 828 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
 829 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
 830 extern gint32 InterlockedIncrement(volatile gint32 *dest);
 831 extern gint32 InterlockedDecrement(volatile gint32 *dest);
 832 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
 833 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
 834 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
 835
 836 #if defined(__hpux) && !defined(__GNUC__)
 837 #define WAPI_ATOMIC_ASM
 838 #endif
 839
 840 #endif
 841
 842 #endif /* _WAPI_ATOMIC_H_ */