mono/utils/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  * Copyright 2012 Xamarin Inc
   9  */
  10
  11 #ifndef _WAPI_ATOMIC_H_
  12 #define _WAPI_ATOMIC_H_
  13
  14 #include "config.h"
  15 #include <glib.h>
  16
  17 #ifdef ENABLE_EXTENSION_MODULE
  18 #include "../../../mono-extensions/mono/utils/atomic.h"
  19 #endif
  20
  21 /* On Windows, we always use the functions provided by the Windows API. */
  22 #if defined(__WIN32__) || defined(_WIN32)
  23
  24 #include <windows.h>
  25
  26 /* mingw is missing InterlockedCompareExchange64 () from winbase.h */
  27 #if HAVE_DECL_INTERLOCKEDCOMPAREEXCHANGE64==0
  28 static inline gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
  29 {
  30         return __sync_val_compare_and_swap (dest, comp, exch);
  31 }
  32 #endif
  33
  34 /* Prefer GCC atomic ops if the target supports it (see configure.in). */
  35 #elif defined(USE_GCC_ATOMIC_OPS)
  36
  37 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  38                                                 gint32 exch, gint32 comp)
  39 {
  40         return __sync_val_compare_and_swap (dest, comp, exch);
  41 }
  42
  43 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  44 {
  45         return __sync_val_compare_and_swap (dest, comp, exch);
  46 }
  47
  48 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  49 {
  50         return __sync_add_and_fetch (val, 1);
  51 }
  52
  53 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  54 {
  55         return __sync_add_and_fetch (val, -1);
  56 }
  57
  58 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  59 {
  60         gint32 old_val;
  61         do {
  62                 old_val = *val;
  63         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
  64         return old_val;
  65 }
  66
  67 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
  68                                                   gpointer new_val)
  69 {
  70         gpointer old_val;
  71         do {
  72                 old_val = *val;
  73         } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
  74         return old_val;
  75 }
  76
  77 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
  78 {
  79         return __sync_fetch_and_add (val, add);
  80 }
  81
  82 #if defined (TARGET_OSX)
  83 #define BROKEN_64BIT_ATOMICS_INTRINSIC 1
  84 #endif
  85
  86 #if !defined (BROKEN_64BIT_ATOMICS_INTRINSIC)
  87
  88 static inline gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
  89 {
  90         return __sync_val_compare_and_swap (dest, comp, exch);
  91 }
  92
  93 #endif
  94
  95 #elif (defined(sparc) || defined (__sparc__)) && defined(__GNUC__)
  96
  97 G_GNUC_UNUSED
  98 static inline gint32 InterlockedCompareExchange(volatile gint32 *_dest, gint32 _exch, gint32 _comp)
  99 {
 100        register volatile gint32 *dest asm("g1") = _dest;
 101        register gint32 comp asm("o4") = _comp;
 102        register gint32 exch asm("o5") = _exch;
 103
 104        __asm__ __volatile__(
 105                /* cas [%%g1], %%o4, %%o5 */
 106                ".word 0xdbe0500c"
 107                : "=r" (exch)
 108                : "0" (exch), "r" (dest), "r" (comp)
 109                : "memory");
 110
 111        return exch;
 112 }
 113
 114 G_GNUC_UNUSED
 115 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *_dest, gpointer _exch, gpointer _comp)
 116 {
 117        register volatile gpointer *dest asm("g1") = _dest;
 118        register gpointer comp asm("o4") = _comp;
 119        register gpointer exch asm("o5") = _exch;
 120
 121        __asm__ __volatile__(
 122 #ifdef SPARCV9
 123                /* casx [%%g1], %%o4, %%o5 */
 124                ".word 0xdbf0500c"
 125 #else
 126                /* cas [%%g1], %%o4, %%o5 */
 127                ".word 0xdbe0500c"
 128 #endif
 129                : "=r" (exch)
 130                : "0" (exch), "r" (dest), "r" (comp)
 131                : "memory");
 132
 133        return exch;
 134 }
 135
 136 G_GNUC_UNUSED
 137 static inline gint32 InterlockedIncrement(volatile gint32 *_dest)
 138 {
 139        register volatile gint32 *dest asm("g1") = _dest;
 140        register gint32 tmp asm("o4");
 141        register gint32 ret asm("o5");
 142
 143        __asm__ __volatile__(
 144                "1:     ld      [%%g1], %%o4\n\t"
 145                "       add     %%o4, 1, %%o5\n\t"
 146                /*      cas     [%%g1], %%o4, %%o5 */
 147                "       .word   0xdbe0500c\n\t"
 148                "       cmp     %%o4, %%o5\n\t"
 149                "       bne     1b\n\t"
 150                "        add    %%o5, 1, %%o5"
 151                : "=&r" (tmp), "=&r" (ret)
 152                : "r" (dest)
 153                : "memory", "cc");
 154
 155         return ret;
 156 }
 157
 158 G_GNUC_UNUSED
 159 static inline gint32 InterlockedDecrement(volatile gint32 *_dest)
 160 {
 161        register volatile gint32 *dest asm("g1") = _dest;
 162        register gint32 tmp asm("o4");
 163        register gint32 ret asm("o5");
 164
 165        __asm__ __volatile__(
 166                "1:     ld      [%%g1], %%o4\n\t"
 167                "       sub     %%o4, 1, %%o5\n\t"
 168                /*      cas     [%%g1], %%o4, %%o5 */
 169                "       .word   0xdbe0500c\n\t"
 170                "       cmp     %%o4, %%o5\n\t"
 171                "       bne     1b\n\t"
 172                "        sub    %%o5, 1, %%o5"
 173                : "=&r" (tmp), "=&r" (ret)
 174                : "r" (dest)
 175                : "memory", "cc");
 176
 177         return ret;
 178 }
 179
 180 G_GNUC_UNUSED
 181 static inline gint32 InterlockedExchange(volatile gint32 *_dest, gint32 exch)
 182 {
 183        register volatile gint32 *dest asm("g1") = _dest;
 184        register gint32 tmp asm("o4");
 185        register gint32 ret asm("o5");
 186
 187        __asm__ __volatile__(
 188                "1:     ld      [%%g1], %%o4\n\t"
 189                "       mov     %3, %%o5\n\t"
 190                /*      cas     [%%g1], %%o4, %%o5 */
 191                "       .word   0xdbe0500c\n\t"
 192                "       cmp     %%o4, %%o5\n\t"
 193                "       bne     1b\n\t"
 194                "        nop"
 195                : "=&r" (tmp), "=&r" (ret)
 196                : "r" (dest), "r" (exch)
 197                : "memory", "cc");
 198
 199         return ret;
 200 }
 201
 202 G_GNUC_UNUSED
 203 static inline gpointer InterlockedExchangePointer(volatile gpointer *_dest, gpointer exch)
 204 {
 205        register volatile gpointer *dest asm("g1") = _dest;
 206        register gpointer tmp asm("o4");
 207        register gpointer ret asm("o5");
 208
 209        __asm__ __volatile__(
 210 #ifdef SPARCV9
 211                "1:     ldx     [%%g1], %%o4\n\t"
 212 #else
 213                "1:     ld      [%%g1], %%o4\n\t"
 214 #endif
 215                "       mov     %3, %%o5\n\t"
 216 #ifdef SPARCV9
 217                /*      casx    [%%g1], %%o4, %%o5 */
 218                "       .word   0xdbf0500c\n\t"
 219 #else
 220                /*      cas     [%%g1], %%o4, %%o5 */
 221                "       .word   0xdbe0500c\n\t"
 222 #endif
 223                "       cmp     %%o4, %%o5\n\t"
 224                "       bne     1b\n\t"
 225                "        nop"
 226                : "=&r" (tmp), "=&r" (ret)
 227                : "r" (dest), "r" (exch)
 228                : "memory", "cc");
 229
 230         return ret;
 231 }
 232
 233 G_GNUC_UNUSED
 234 static inline gint32 InterlockedExchangeAdd(volatile gint32 *_dest, gint32 add)
 235 {
 236        register volatile gint32 *dest asm("g1") = _dest;
 237        register gint32 tmp asm("o4");
 238        register gint32 ret asm("o5");
 239
 240        __asm__ __volatile__(
 241                "1:     ld      [%%g1], %%o4\n\t"
 242                "       add     %%o4, %3, %%o5\n\t"
 243                /*      cas     [%%g1], %%o4, %%o5 */
 244                "       .word   0xdbe0500c\n\t"
 245                "       cmp     %%o4, %%o5\n\t"
 246                "       bne     1b\n\t"
 247                "        add    %%o5, %3, %%o5"
 248                : "=&r" (tmp), "=&r" (ret)
 249                : "r" (dest), "r" (add)
 250                : "memory", "cc");
 251
 252         return ret;
 253 }
 254
 255 #elif __s390x__
 256
 257 static inline gint32
 258 InterlockedCompareExchange(volatile gint32 *dest,
 259                            gint32 exch, gint32 comp)
 260 {
 261         gint32 old;
 262
 263         __asm__ __volatile__ ("\tLA\t1,%0\n"
 264                               "\tLR\t%1,%3\n"
 265                               "\tCS\t%1,%2,0(1)\n"
 266                               : "+m" (*dest), "=&r" (old)
 267                               : "r" (exch), "r" (comp)
 268                               : "1", "cc");
 269         return(old);
 270 }
 271
 272 static inline gpointer
 273 InterlockedCompareExchangePointer(volatile gpointer *dest,
 274                                   gpointer exch,
 275                                   gpointer comp)
 276 {
 277         gpointer old;
 278
 279         __asm__ __volatile__ ("\tLA\t1,%0\n"
 280                               "\tLGR\t%1,%3\n"
 281                               "\tCSG\t%1,%2,0(1)\n"
 282                               : "+m" (*dest), "=&r" (old)
 283                               : "r" (exch), "r" (comp)
 284                               : "1", "cc");
 285
 286         return(old);
 287 }
 288
 289 static inline gint32
 290 InterlockedIncrement(volatile gint32 *val)
 291 {
 292         gint32 tmp;
 293
 294         __asm__ __volatile__ ("\tLA\t2,%1\n"
 295                               "0:\tLGF\t%0,%1\n"
 296                               "\tLGFR\t1,%0\n"
 297                               "\tAGHI\t1,1\n"
 298                               "\tCS\t%0,1,0(2)\n"
 299                               "\tJNZ\t0b\n"
 300                               "\tLGFR\t%0,1"
 301                               : "=r" (tmp), "+m" (*val)
 302                               : : "1", "2", "cc");
 303
 304         return(tmp);
 305 }
 306
 307 static inline gint32
 308 InterlockedDecrement(volatile gint32 *val)
 309 {
 310         gint32 tmp;
 311
 312         __asm__ __volatile__ ("\tLA\t2,%1\n"
 313                               "0:\tLGF\t%0,%1\n"
 314                               "\tLGFR\t1,%0\n"
 315                               "\tAGHI\t1,-1\n"
 316                               "\tCS\t%0,1,0(2)\n"
 317                               "\tJNZ\t0b\n"
 318                               "\tLGFR\t%0,1"
 319                               : "=r" (tmp), "+m" (*val)
 320                               : : "1", "2", "cc");
 321
 322         return(tmp);
 323 }
 324
 325 static inline gint32
 326 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 327 {
 328         gint32 ret;
 329
 330         __asm__ __volatile__ ("\tLA\t1,%0\n"
 331                               "0:\tL\t%1,%0\n"
 332                               "\tCS\t%1,%2,0(1)\n"
 333                               "\tJNZ\t0b"
 334                               : "+m" (*val), "=&r" (ret)
 335                               : "r" (new_val)
 336                               : "1", "cc");
 337
 338         return(ret);
 339 }
 340
 341 static inline gpointer
 342 InterlockedExchangePointer(volatile gpointer *val, gpointer new_val)
 343 {
 344         gpointer ret;
 345
 346         __asm__ __volatile__ ("\tLA\t1,%0\n"
 347                               "0:\tLG\t%1,%0\n"
 348                               "\tCSG\t%1,%2,0(1)\n"
 349                               "\tJNZ\t0b"
 350                               : "+m" (*val), "=&r" (ret)
 351                               : "r" (new_val)
 352                               : "1", "cc");
 353
 354         return(ret);
 355 }
 356
 357 static inline gint32
 358 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 359 {
 360         gint32 ret;
 361
 362         __asm__ __volatile__ ("\tLA\t2,%1\n"
 363                               "0:\tLGF\t%0,%1\n"
 364                               "\tLGFR\t1,%0\n"
 365                               "\tAGR\t1,%2\n"
 366                               "\tCS\t%0,1,0(2)\n"
 367                               "\tJNZ\t0b"
 368                               : "=&r" (ret), "+m" (*val)
 369                               : "r" (add)
 370                               : "1", "2", "cc");
 371
 372         return(ret);
 373 }
 374
 375 #elif defined(__ia64__)
 376
 377 #ifdef __INTEL_COMPILER
 378 #include <ia64intrin.h>
 379 #endif
 380
 381 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
 382                                                 gint32 exch, gint32 comp)
 383 {
 384         gint32 old;
 385         guint64 real_comp;
 386
 387 #ifdef __INTEL_COMPILER
 388         old = _InterlockedCompareExchange (dest, exch, comp);
 389 #else
 390         /* cmpxchg4 zero extends the value read from memory */
 391         real_comp = (guint64)(guint32)comp;
 392         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 393                                   "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
 394                                   : "=r" (old) : "r" (dest), "r" (real_comp), "r" (exch));
 395 #endif
 396
 397         return(old);
 398 }
 399
 400 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
 401                                                 gpointer exch, gpointer comp)
 402 {
 403         gpointer old;
 404
 405 #ifdef __INTEL_COMPILER
 406         old = _InterlockedCompareExchangePointer (dest, exch, comp);
 407 #else
 408         asm volatile ("mov ar.ccv = %2 ;;\n\t"
 409                                   "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
 410                                   : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
 411 #endif
 412
 413         return(old);
 414 }
 415
 416 static inline gint32 InterlockedIncrement(gint32 volatile *val)
 417 {
 418 #ifdef __INTEL_COMPILER
 419         return _InterlockedIncrement (val);
 420 #else
 421         gint32 old;
 422
 423         do {
 424                 old = *val;
 425         } while (InterlockedCompareExchange (val, old + 1, old) != old);
 426
 427         return old + 1;
 428 #endif
 429 }
 430
 431 static inline gint32 InterlockedDecrement(gint32 volatile *val)
 432 {
 433 #ifdef __INTEL_COMPILER
 434         return _InterlockedDecrement (val);
 435 #else
 436         gint32 old;
 437
 438         do {
 439                 old = *val;
 440         } while (InterlockedCompareExchange (val, old - 1, old) != old);
 441
 442         return old - 1;
 443 #endif
 444 }
 445
 446 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
 447 {
 448 #ifdef __INTEL_COMPILER
 449         return _InterlockedExchange (dest, new_val);
 450 #else
 451         gint32 res;
 452
 453         do {
 454                 res = *dest;
 455         } while (InterlockedCompareExchange (dest, new_val, res) != res);
 456
 457         return res;
 458 #endif
 459 }
 460
 461 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
 462 {
 463 #ifdef __INTEL_COMPILER
 464         return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
 465 #else
 466         gpointer res;
 467
 468         do {
 469                 res = *dest;
 470         } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
 471
 472         return res;
 473 #endif
 474 }
 475
 476 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
 477 {
 478         gint32 old;
 479
 480 #ifdef __INTEL_COMPILER
 481         old = _InterlockedExchangeAdd (val, add);
 482 #else
 483         do {
 484                 old = *val;
 485         } while (InterlockedCompareExchange (val, old + add, old) != old);
 486
 487         return old;
 488 #endif
 489 }
 490
 491 #else
 492
 493 #define WAPI_NO_ATOMIC_ASM
 494
 495 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
 496 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
 497 extern gint32 InterlockedIncrement(volatile gint32 *dest);
 498 extern gint32 InterlockedDecrement(volatile gint32 *dest);
 499 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
 500 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
 501 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
 502
 503 #endif
 504
 505 #if defined (WAPI_NO_ATOMIC_ASM) || defined (BROKEN_64BIT_ATOMICS_INTRINSIC)
 506
 507 extern gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp);
 508
 509 #endif
 510
 511 #endif /* _WAPI_ATOMIC_H_ */