mono/utils/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  * Copyright 2012 Xamarin Inc
   9  */
  10
  11 #ifndef _WAPI_ATOMIC_H_
  12 #define _WAPI_ATOMIC_H_
  13
  14 #if defined(__NetBSD__)
  15 #include <sys/param.h>
  16
  17 #if __NetBSD_Version__ > 499004000
  18 #include <sys/atomic.h>
  19 #define HAVE_ATOMIC_OPS
  20 #endif
  21
  22 #endif
  23
  24 #include "config.h"
  25 #include <glib.h>
  26
  27 /* On Windows, we always use the functions provided by the Windows API. */
  28 #if defined(__WIN32__) || defined(_WIN32)
  29
  30 #include <windows.h>
  31
  32 /* Prefer GCC atomic ops if the target supports it (see configure.in). */
  33 #elif defined(USE_GCC_ATOMIC_OPS)
  34
  35 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  36                                                 gint32 exch, gint32 comp)
  37 {
  38         return __sync_val_compare_and_swap (dest, comp, exch);
  39 }
  40
  41 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  42 {
  43         return __sync_val_compare_and_swap (dest, comp, exch);
  44 }
  45
  46 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  47 {
  48         return __sync_add_and_fetch (val, 1);
  49 }
  50
  51 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  52 {
  53         return __sync_add_and_fetch (val, -1);
  54 }
  55
  56 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  57 {
  58         gint32 old_val;
  59         do {
  60                 old_val = *val;
  61         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
  62         return old_val;
  63 }
  64
  65 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
  66                                                   gpointer new_val)
  67 {
  68         gpointer old_val;
  69         do {
  70                 old_val = *val;
  71         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
  72         return old_val;
  73 }
  74
  75 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
  76 {
  77         return __sync_fetch_and_add (val, add);
  78 }
  79
  80 #elif defined(__NetBSD__) && defined(HAVE_ATOMIC_OPS)
  81
  82 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  83        gint32 exch, gint32 comp)
  84 {
  85        return atomic_cas_32((uint32_t*)dest, comp, exch);
  86 }
  87
  88 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  89 {
  90        return atomic_cas_ptr(dest, comp, exch);
  91 }
  92
  93 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  94 {
  95        return atomic_inc_32_nv((uint32_t*)val);
  96 }
  97
  98 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  99 {
 100        return atomic_dec_32_nv((uint32_t*)val);
 101 }
 102
 103 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 104 {
 105        return atomic_swap_32((uint32_t*)val, new_val);
 106 }
 107
 108 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
 109                gpointer new_val)
 110 {
 111        return atomic_swap_ptr(val, new_val);
 112 }
 113
 114 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 115 {
 116        return atomic_add_32_nv((uint32_t*)val, add) - add;
 117 }
 118
 119 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
 120
 121 G_GNUC_UNUSED
 122 static inline gint32 InterlockedCompareExchange(volatile gint32 *_dest, gint32 _exch, gint32 _comp)
 123 {
 124        register volatile gint32 *dest asm("g1") = _dest;
 125        register gint32 comp asm("o4") = _comp;
 126        register gint32 exch asm("o5") = _exch;
 127
 128        __asm__ __volatile__(
 129                /* cas [%%g1], %%o4, %%o5 */
 130                ".word 0xdbe0500c"
 131                : "=r" (exch)
 132                : "0" (exch), "r" (dest), "r" (comp)
 133                : "memory");
 134
 135        return exch;
 136 }
 137
 138 G_GNUC_UNUSED
 139 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *_dest, gpointer _exch, gpointer _comp)
 140 {
 141        register volatile gpointer *dest asm("g1") = _dest;
 142        register gpointer comp asm("o4") = _comp;
 143        register gpointer exch asm("o5") = _exch;
 144
 145        __asm__ __volatile__(
 146 #ifdef SPARCV9
 147                /* casx [%%g1], %%o4, %%o5 */
 148                ".word 0xdbf0500c"
 149 #else
 150                /* cas [%%g1], %%o4, %%o5 */
 151                ".word 0xdbe0500c"
 152 #endif
 153                : "=r" (exch)
 154                : "0" (exch), "r" (dest), "r" (comp)
 155                : "memory");
 156
 157        return exch;
 158 }
 159
 160 G_GNUC_UNUSED
 161 static inline gint32 InterlockedIncrement(volatile gint32 *_dest)
 162 {
 163        register volatile gint32 *dest asm("g1") = _dest;
 164        register gint32 tmp asm("o4");
 165        register gint32 ret asm("o5");
 166
 167        __asm__ __volatile__(
 168                "1:     ld      [%%g1], %%o4\n\t"
 169                "       add     %%o4, 1, %%o5\n\t"
 170                /*      cas     [%%g1], %%o4, %%o5 */
 171                "       .word   0xdbe0500c\n\t"
 172                "       cmp     %%o4, %%o5\n\t"
 173                "       bne     1b\n\t"
 174                "        add    %%o5, 1, %%o5"
 175                : "=&r" (tmp), "=&r" (ret)
 176                : "r" (dest)
 177                : "memory", "cc");
 178
 179         return ret;
 180 }
 181
 182 G_GNUC_UNUSED
 183 static inline gint32 InterlockedDecrement(volatile gint32 *_dest)
 184 {
 185        register volatile gint32 *dest asm("g1") = _dest;
 186        register gint32 tmp asm("o4");
 187        register gint32 ret asm("o5");
 188
 189        __asm__ __volatile__(
 190                "1:     ld      [%%g1], %%o4\n\t"
 191                "       sub     %%o4, 1, %%o5\n\t"
 192                /*      cas     [%%g1], %%o4, %%o5 */
 193                "       .word   0xdbe0500c\n\t"
 194                "       cmp     %%o4, %%o5\n\t"
 195                "       bne     1b\n\t"
 196                "        sub    %%o5, 1, %%o5"
 197                : "=&r" (tmp), "=&r" (ret)
 198                : "r" (dest)
 199                : "memory", "cc");
 200
 201         return ret;
 202 }
 203
 204 G_GNUC_UNUSED
 205 static inline gint32 InterlockedExchange(volatile gint32 *_dest, gint32 exch)
 206 {
 207        register volatile gint32 *dest asm("g1") = _dest;
 208        register gint32 tmp asm("o4");
 209        register gint32 ret asm("o5");
 210
 211        __asm__ __volatile__(
 212                "1:     ld      [%%g1], %%o4\n\t"
 213                "       mov     %3, %%o5\n\t"
 214                /*      cas     [%%g1], %%o4, %%o5 */
 215                "       .word   0xdbe0500c\n\t"
 216                "       cmp     %%o4, %%o5\n\t"
 217                "       bne     1b\n\t"
 218                "        nop"
 219                : "=&r" (tmp), "=&r" (ret)
 220                : "r" (dest), "r" (exch)
 221                : "memory", "cc");
 222
 223         return ret;
 224 }
 225
 226 G_GNUC_UNUSED
 227 static inline gpointer InterlockedExchangePointer(volatile gpointer *_dest, gpointer exch)
 228 {
 229        register volatile gpointer *dest asm("g1") = _dest;
 230        register gpointer tmp asm("o4");
 231        register gpointer ret asm("o5");
 232
 233        __asm__ __volatile__(
 234 #ifdef SPARCV9
 235                "1:     ldx     [%%g1], %%o4\n\t"
 236 #else
 237                "1:     ld      [%%g1], %%o4\n\t"
 238 #endif
 239                "       mov     %3, %%o5\n\t"
 240 #ifdef SPARCV9
 241                /*      casx    [%%g1], %%o4, %%o5 */
 242                "       .word   0xdbf0500c\n\t"
 243 #else
 244                /*      cas     [%%g1], %%o4, %%o5 */
 245                "       .word   0xdbe0500c\n\t"
 246 #endif
 247                "       cmp     %%o4, %%o5\n\t"
 248                "       bne     1b\n\t"
 249                "        nop"
 250                : "=&r" (tmp), "=&r" (ret)
 251                : "r" (dest), "r" (exch)
 252                : "memory", "cc");
 253
 254         return ret;
 255 }
 256
 257 G_GNUC_UNUSED
 258 static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
 259 {
 260        register volatile gint32 *dest asm("g1") = _dest;
 261        register gint32 tmp asm("o4");
 262        register gint32 ret asm("o5");
 263
 264        __asm__ __volatile__(
 265                "1:     ld      [%%g1], %%o4\n\t"
 266                "       add     %%o4, %3, %%o5\n\t"
 267                /*      cas     [%%g1], %%o4, %%o5 */
 268                "       .word   0xdbe0500c\n\t"
 269                "       cmp     %%o4, %%o5\n\t"
 270                "       bne     1b\n\t"
 271                "        add    %%o5, %3, %%o5"
 272                : "=&r" (tmp), "=&r" (ret)
 273                : "r" (dest), "r" (add)
 274                : "memory", "cc");
 275
 276         return ret;
 277 }
 278
 279 #elif __s390x__
 280
 281 static inline gint32
 282 InterlockedCompareExchange(volatile gint32 *dest,
 283                            gint32 exch, gint32 comp)
 284 {
 285         gint32 old;
 286
 287         __asm__ __volatile__ ("\tLA\t1,%0\n"
 288                               "\tLR\t%1,%3\n"
 289                               "\tCS\t%1,%2,0(1)\n"
 290                               : "+m" (*dest), "=&r" (old)
 291                               : "r" (exch), "r" (comp)
 292                               : "1", "cc");
 293         return(old);
 294 }
 295
 296 static inline gpointer
 297 InterlockedCompareExchangePointer(volatile gpointer *dest,
 298                                   gpointer exch,
 299                                   gpointer comp)
 300 {
 301         gpointer old;
 302
 303         __asm__ __volatile__ ("\tLA\t1,%0\n"
 304                               "\tLGR\t%1,%3\n"
 305                               "\tCSG\t%1,%2,0(1)\n"
 306                               : "+m" (*dest), "=&r" (old)
 307                               : "r" (exch), "r" (comp)
 308                               : "1", "cc");
 309
 310         return(old);
 311 }
 312
 313 static inline gint32
 314 InterlockedIncrement(volatile gint32 *val)
 315 {
 316         gint32 tmp;
 317
 318         __asm__ __volatile__ ("\tLA\t2,%1\n"
 319                               "0:\tLGF\t%0,%1\n"
 320                               "\tLGFR\t1,%0\n"
 321                               "\tAGHI\t1,1\n"
 322                               "\tCS\t%0,1,0(2)\n"
 323                               "\tJNZ\t0b\n"
 324                               "\tLGFR\t%0,1"
 325                               : "=r" (tmp), "+m" (*val)
 326                               : : "1", "2", "cc");
 327
 328         return(tmp);
 329 }
 330
 331 static inline gint32
 332 InterlockedDecrement(volatile gint32 *val)
 333 {
 334         gint32 tmp;
 335
 336         __asm__ __volatile__ ("\tLA\t2,%1\n"
 337                               "0:\tLGF\t%0,%1\n"
 338                               "\tLGFR\t1,%0\n"
 339                               "\tAGHI\t1,-1\n"
 340                               "\tCS\t%0,1,0(2)\n"
 341                               "\tJNZ\t0b\n"
 342                               "\tLGFR\t%0,1"
 343                               : "=r" (tmp), "+m" (*val)
 344                               : : "1", "2", "cc");
 345
 346         return(tmp);
 347 }
 348
 349 static inline gint32
 350 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 351 {
 352         gint32 ret;
 353
 354         __asm__ __volatile__ ("\tLA\t1,%0\n"
 355                               "0:\tL\t%1,%0\n"
 356                               "\tCS\t%1,%2,0(1)\n"
 357                               "\tJNZ\t0b"
 358                               : "+m" (*val), "=&r" (ret)
 359                               : "r" (new_val)
 360                               : "1", "cc");
 361
 362         return(ret);
 363 }
 364
 365 static inline gpointer
 366 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 367 {
 368         gpointer ret;
 369
 370         __asm__ __volatile__ ("\tLA\t1,%0\n"
 371                               "0:\tLG\t%1,%0\n"
 372                               "\tCSG\t%1,%2,0(1)\n"
 373                               "\tJNZ\t0b"
 374                               : "+m" (*val), "=&r" (ret)
 375                               : "r" (new_val)
 376                               : "1", "cc");
 377
 378         return(ret);
 379 }
 380
 381 static inline gint32
 382 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 383 {
 384         gint32 ret;
 385
 386         __asm__ __volatile__ ("\tLA\t2,%1\n"
 387                               "0:\tLGF\t%0,%1\n"
 388                               "\tLGFR\t1,%0\n"
 389                               "\tAGR\t1,%2\n"
 390                               "\tCS\t%0,1,0(2)\n"
 391                               "\tJNZ\t0b"
 392                               : "=&r" (ret), "+m" (*val)
 393                               : "r" (add)
 394                               : "1", "2", "cc");
 395
 396         return(ret);
 397 }
 398
 399 #elif defined(__ia64__)
 400
 401 #ifdef __INTEL_COMPILER
 402 #include <ia64intrin.h>
 403 #endif
 404
 405 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
 406                                                 gint32 exch, gint32 comp)
 407 {
 408         gint32 old;
 409         guint64 real_comp;
 410
 411 #ifdef __INTEL_COMPILER
 412         old = _InterlockedCompareExchange (dest, exch, comp);
 413 #else
 414         /* cmpxchg4 zero extends the value read from memory */
 415         real_comp = (guint64)(guint32)comp;
 416         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 417                                   "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
 418                                   : "=r" (old) : "r" (dest), "r" (real_comp), "r" (exch));
 419 #endif
 420
 421         return(old);
 422 }
 423
 424 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
 425                                                 gpointer exch, gpointer comp)
 426 {
 427         gpointer old;
 428
 429 #ifdef __INTEL_COMPILER
 430         old = _InterlockedCompareExchangePointer (dest, exch, comp);
 431 #else
 432         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 433                                   "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
 434                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
 435 #endif
 436
 437         return(old);
 438 }
 439
 440 static inline gint32 InterlockedIncrement(gint32 volatile *val)
 441 {
 442 #ifdef __INTEL_COMPILER
 443         return _InterlockedIncrement (val);
 444 #else
 445         gint32 old;
 446
 447         do {
 448                 old = *val;
 449         } while (InterlockedCompareExchange (val, old + 1, old) != old);
 450
 451         return old + 1;
 452 #endif
 453 }
 454
 455 static inline gint32 InterlockedDecrement(gint32 volatile *val)
 456 {
 457 #ifdef __INTEL_COMPILER
 458         return _InterlockedDecrement (val);
 459 #else
 460         gint32 old;
 461
 462         do {
 463                 old = *val;
 464         } while (InterlockedCompareExchange (val, old - 1, old) != old);
 465
 466         return old - 1;
 467 #endif
 468 }
 469
 470 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
 471 {
 472 #ifdef __INTEL_COMPILER
 473         return _InterlockedExchange (dest, new_val);
 474 #else
 475         gint32 res;
 476
 477         do {
 478                 res = *dest;
 479         } while (InterlockedCompareExchange (dest, new_val, res) != res);
 480
 481         return res;
 482 #endif
 483 }
 484
 485 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
 486 {
 487 #ifdef __INTEL_COMPILER
 488         return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
 489 #else
 490         gpointer res;
 491
 492         do {
 493                 res = *dest;
 494         } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
 495
 496         return res;
 497 #endif
 498 }
 499
 500 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
 501 {
 502         gint32 old;
 503
 504 #ifdef __INTEL_COMPILER
 505         old = _InterlockedExchangeAdd (val, add);
 506 #else
 507         do {
 508                 old = *val;
 509         } while (InterlockedCompareExchange (val, old + add, old) != old);
 510
 511         return old;
 512 #endif
 513 }
 514
 515 #else
 516
 517 #define WAPI_NO_ATOMIC_ASM
 518
 519 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
 520 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
 521 extern gint32 InterlockedIncrement(volatile gint32 *dest);
 522 extern gint32 InterlockedDecrement(volatile gint32 *dest);
 523 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
 524 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
 525 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
 526
 527 #endif
 528
 529 #endif /* _WAPI_ATOMIC_H_ */