mono/io-layer/atomic.h

   1 /*
   2  * atomic.h:  Atomic operations
   3  *
   4  * Author:
   5  *      Dick Porter (dick@ximian.com)
   6  *
   7  * (C) 2002 Ximian, Inc.
   8  */
   9
  10 #ifndef _WAPI_ATOMIC_H_
  11 #define _WAPI_ATOMIC_H_
  12
  13 #include <glib.h>
  14
  15 #include "mono/io-layer/wapi.h"
  16
  17 #if defined(__i386__) || defined(__x86_64__)
  18 #define WAPI_ATOMIC_ASM
  19
  20 /*
  21  * NB: The *Pointer() functions here assume that
  22  * sizeof(pointer)==sizeof(gint32)
  23  *
  24  * NB2: These asm functions assume 486+ (some of the opcodes dont
  25  * exist on 386).  If this becomes an issue, we can get configure to
  26  * fall back to the non-atomic C versions of these calls.
  27  */
  28
  29 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
  30                                                 gint32 exch, gint32 comp)
  31 {
  32         gint32 old;
  33
  34         __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
  35                               : "=m" (*dest), "=a" (old)
  36                               : "r" (exch), "m" (*dest), "a" (comp));
  37         return(old);
  38 }
  39
  40 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
  41 {
  42         gpointer old;
  43
  44         __asm__ __volatile__ ("lock; "
  45 #ifdef __x86_64__
  46                               "cmpxchgq"
  47 #else
  48                               "cmpxchgl"
  49 #endif
  50                               " %2, %0"
  51                               : "=m" (*dest), "=a" (old)
  52                               : "r" (exch), "m" (*dest), "a" (comp));
  53
  54         return(old);
  55 }
  56
  57 static inline gint32 InterlockedIncrement(volatile gint32 *val)
  58 {
  59         gint32 tmp;
  60
  61         __asm__ __volatile__ ("lock; xaddl %0, %1"
  62                               : "=r" (tmp), "=m" (*val)
  63                               : "0" (1), "m" (*val));
  64
  65         return(tmp+1);
  66 }
  67
  68 static inline gint32 InterlockedDecrement(volatile gint32 *val)
  69 {
  70         gint32 tmp;
  71
  72         __asm__ __volatile__ ("lock; xaddl %0, %1"
  73                               : "=r" (tmp), "=m" (*val)
  74                               : "0" (-1), "m" (*val));
  75
  76         return(tmp-1);
  77 }
  78
  79 /*
  80  * See
  81  * http://msdn.microsoft.com/library/en-us/dnmag00/html/win320700.asp?frame=true
  82  * for the reasons for using cmpxchg and a loop here.
  83  *
  84  * That url is no longer valid, but it's still in the google cache at the
  85  * moment: http://www.google.com/search?q=cache:http://msdn.microsoft.com/library/en-us/dnmag00/html/win320700.asp?frame=true
  86  *
  87  * For the time being, http://msdn.microsoft.com/msdnmag/issues/0700/Win32/
  88  * might work.  Bet it will change soon enough though.
  89  */
  90 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
  91 {
  92         gint32 ret;
  93
  94         __asm__ __volatile__ ("1:; lock; cmpxchgl %2, %0; jne 1b"
  95                               : "=m" (*val), "=a" (ret)
  96                               : "r" (new_val), "m" (*val), "a" (*val));
  97
  98         return(ret);
  99 }
 100
 101 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
 102                                                   gpointer new_val)
 103 {
 104         gpointer ret;
 105
 106         __asm__ __volatile__ ("1:; lock; "
 107 #ifdef __x86_64__
 108                               "cmpxchgq"
 109 #else
 110                               "cmpxchgl"
 111 #endif
 112                               " %2, %0; jne 1b"
 113                               : "=m" (*val), "=a" (ret)
 114                               : "r" (new_val), "m" (*val), "a" (*val));
 115
 116         return(ret);
 117 }
 118
 119 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 120 {
 121         gint32 ret;
 122
 123         __asm__ __volatile__ ("lock; xaddl %0, %1"
 124                               : "=r" (ret), "=m" (*val)
 125                               : "0" (add), "m" (*val));
 126
 127         return(ret);
 128 }
 129
 130 #elif defined(sparc) || defined (__sparc__)
 131 #define WAPI_ATOMIC_ASM
 132
 133 #ifdef __GNUC__
 134 #define BEGIN_SPIN(tmp,lock) \
 135 __asm__ __volatile__("1:        ldstub [%1],%0\n\t"  \
 136                              "          cmp %0, 0\n\t" \
 137                              "          bne 1b\n\t" \
 138                              "          nop" \
 139                              : "=&r" (tmp) \
 140                              : "r" (&lock) \
 141                              : "memory");
 142
 143 #define END_SPIN(lock) \
 144 __asm__ __volatile__("stb       %%g0, [%0]"  \
 145                       : /* no outputs */ \
 146                       : "r" (&lock)\
 147                       : "memory");
 148 #else
 149 static inline void begin_spin(volatile unsigned char *lock)
 150 {
 151         asm("1: ldstub [%i0], %l0");
 152         asm("cmp %l0,0");
 153         asm("bne 1b");
 154         asm("nop");
 155 }
 156 #define BEGIN_SPIN(tmp,lock) begin_spin(&lock);
 157 #define END_SPIN(lock) ((lock) = 0);
 158 #endif
 159
 160 extern volatile unsigned char _wapi_sparc_lock;
 161
 162 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp)
 163 {
 164         int tmp;
 165         gint32 old;
 166
 167         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 168
 169         old = *dest;
 170         if (old==comp) {
 171                 *dest=exch;
 172         }
 173
 174         END_SPIN(_wapi_sparc_lock)
 175
 176         return(old);
 177 }
 178
 179 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
 180 {
 181         int tmp;
 182         gpointer old;
 183
 184         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 185
 186         old = *dest;
 187         if (old==comp) {
 188                 *dest=exch;
 189         }
 190
 191         END_SPIN(_wapi_sparc_lock)
 192
 193         return(old);
 194 }
 195
 196 static inline gint32 InterlockedIncrement(volatile gint32 *dest)
 197 {
 198         int tmp;
 199         gint32 ret;
 200
 201         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 202
 203         (*dest)++;
 204         ret = *dest;
 205
 206         END_SPIN(_wapi_sparc_lock)
 207
 208         return(ret);
 209 }
 210
 211 static inline gint32 InterlockedDecrement(volatile gint32 *dest)
 212 {
 213         int tmp;
 214         gint32 ret;
 215
 216         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 217
 218         (*dest)--;
 219         ret = *dest;
 220
 221         END_SPIN(_wapi_sparc_lock)
 222
 223         return(ret);
 224 }
 225
 226 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 227 {
 228         int tmp;
 229         gint32 ret;
 230
 231         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 232
 233         ret = *dest;
 234         *dest = exch;
 235
 236         END_SPIN(_wapi_sparc_lock)
 237
 238         return(ret);
 239 }
 240
 241 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 242 {
 243         int tmp;
 244         gpointer ret;
 245
 246         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 247
 248         ret = *dest;
 249         *dest = exch;
 250
 251         END_SPIN(_wapi_sparc_lock)
 252
 253         return(ret);
 254 }
 255
 256 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 257 {
 258         int tmp;
 259         gint32 ret;
 260
 261         BEGIN_SPIN(tmp,_wapi_sparc_lock)
 262
 263         ret = *dest;
 264         *dest += add;
 265
 266         END_SPIN(_wapi_sparc_lock)
 267
 268         return(ret);
 269 }
 270
 271 #elif __s390__
 272
 273 #define WAPI_ATOMIC_ASM
 274
 275 static inline gint32
 276 InterlockedCompareExchange(volatile gint32 *dest,
 277                            gint32 exch, gint32 comp)
 278 {
 279         gint32 old;
 280
 281         __asm__ __volatile__ ("\tL\t%1,%0\n"
 282                               "\tLA\t1,%0\n"
 283                               "\tCS\t%3,%2,0(1)\n"
 284                               : "=m" (*dest), "=r" (old)
 285                               : "r" (exch), "r" (comp)
 286                               : "1", "cc");
 287         return(old);
 288 }
 289
 290 #define InterlockedCompareExchangePointer InterlockedCompareExchange
 291
 292 static inline gint32
 293 InterlockedIncrement(volatile gint32 *val)
 294 {
 295         gint32 tmp;
 296
 297         __asm__ __volatile__ ("\tLA\t2,%1\n"
 298                               "\tL\t%0,%1\n"
 299                               "\tLR\t1,%0\n"
 300                               "\tAHI\t1,1\n"
 301                               "0:\tCS\t%0,1,0(2)\n"
 302                               "\tJNZ\t0b"
 303                               : "=r" (tmp), "+m" (*val)
 304                               : : "1", "2", "cc");
 305
 306         return(tmp+1);
 307 }
 308
 309 static inline gint32
 310 InterlockedDecrement(volatile gint32 *val)
 311 {
 312         gint32 tmp;
 313
 314         __asm__ __volatile__ ("\tLA\t2,%1\n"
 315                               "\tL\t%0,%1\n"
 316                               "\tLR\t1,%0\n"
 317                               "\tAHI\t1,-1\n"
 318                               "0:\tCS\t%0,1,0(2)\n"
 319                               "\tJNZ\t0b"
 320                               : "=r" (tmp), "+m" (*val)
 321                               : : "1", "2", "cc");
 322
 323         return(tmp-1);
 324 }
 325
 326
 327 static inline gint32
 328 InterlockedExchange(volatile gint32 *val, gint32 new_val)
 329 {
 330         gint32 ret;
 331
 332         __asm__ __volatile__ ("\tLA\t1,%1\n"
 333                               "0:\tL\t%1,%0\n"
 334                               "\tCS\t%1,%2,0(1)\n"
 335                               "\tJNZ\t0b"
 336                               : "+m" (*val), "=r" (ret)
 337                               : "r" (new_val)
 338                               : "1", "cc");
 339
 340         return(ret);
 341 }
 342
 343 #define InterlockedExchangePointer InterlockedExchange
 344
 345 static inline gint32
 346 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
 347 {
 348         gint32 ret;
 349
 350         __asm__ __volatile__ ("\tL\t%0,%1\n"
 351                               "\tLR\t1,%0\n"
 352                               "\tAR\t1,%2\n"
 353                               "\tLA\t2,%1\n"
 354                               "0:\tCS\t%0,1,0(2)\n"
 355                               "\tJNZ\t0b"
 356                               : "=r" (ret), "+m" (*val)
 357                               : "r" (add)
 358                               : "1", "2", "cc");
 359
 360         return(ret);
 361 }
 362
 363 #elif defined(__ppc__) || defined (__powerpc__)
 364 #define WAPI_ATOMIC_ASM
 365
 366 static inline gint32 InterlockedIncrement(volatile gint32 *val)
 367 {
 368         gint32 tmp;
 369
 370         __asm__ __volatile__ ("\n1:\n\t"
 371                               "lwarx  %0, 0, %2\n\t"
 372                               "addi   %1, %0, 1\n\t"
 373                               "stwcx. %1, 0, %2\n\t"
 374                               "bne-   1b"
 375                               : "=&b" (tmp): "r" (tmp), "r" (val): "cc", "memory");
 376         return tmp;
 377 }
 378
 379 static inline gint32 InterlockedDecrement(volatile gint32 *val)
 380 {
 381         gint32 tmp;
 382
 383         __asm__ __volatile__ ("\n1:\n\t"
 384                               "lwarx  %0, 0, %2\n\t"
 385                               "addi   %1, %0, -1\n\t"
 386                               "stwcx. %1, 0, %2\n\t"
 387                               "bne-   1b"
 388                               : "=&b" (tmp) : "r" (tmp), "r" (val): "cc", "memory");
 389         return(tmp);
 390 }
 391
 392 #define InterlockedCompareExchangePointer InterlockedCompareExchange
 393
 394 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
 395                                                 gint32 exch, gint32 comp) {
 396         gint32 tmp = 0;
 397
 398         __asm__ __volatile__ ("\n1:\n\t"
 399                              "lwarx   %0, 0, %1\n\t"
 400                              "cmpw    %2, %3\n\t"
 401                              "bne-    2f\n\t"
 402                              "stwcx.  %4, 0, %1\n\t"
 403                              "bne-    1b\n"
 404                              "2:"
 405                              : "=r" (tmp)
 406                              : "r" (dest), "0" (tmp) ,"r" (comp), "r" (exch): "cc", "memory");
 407         return(tmp);
 408 }
 409
 410 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 411 {
 412         gint32 tmp;
 413
 414         __asm__ __volatile__ ("\n1:\n\t"
 415                               "lwarx  %0, 0, %1\n\t"
 416                               "stwcx. %2, 0, %1\n\t"
 417                               "bne    1b"
 418                               : "=r" (tmp) : "r" (dest), "r" (exch): "cc", "memory");
 419         return(tmp);
 420 }
 421 #define InterlockedExchangePointer InterlockedExchange
 422
 423 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 424 {
 425         gint32 tmp;
 426
 427         __asm__ __volatile__ ("\n1:\n\t"
 428                               "lwarx  %0, 0, %2\n\t"
 429                               "add    %1, %3, %4\n\t"
 430                               "stwcx. %1, 0, %2\n\t"
 431                               "bne    1b"
 432                               : "=r" (tmp), "=r" (add)
 433                               : "r" (dest), "0" (tmp), "1" (add) : "cc", "memory");
 434         return(tmp);
 435 }
 436
 437 #elif defined(__arm__)
 438 #define WAPI_ATOMIC_ASM
 439
 440 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp)
 441 {
 442         int a, b;
 443
 444         __asm__ __volatile__ (    "0:\n\t"
 445                                   "ldr %1, [%2]\n\t"
 446                                   "cmp %1, %4\n\t"
 447                                   "bne 1f\n\t"
 448                                   "swp %0, %3, [%2]\n\t"
 449                                   "cmp %0, %1\n\t"
 450                                   "swpne %3, %0, [%2]\n\t"
 451                                   "bne 0b\n\t"
 452                                   "1:"
 453                                   : "=&r" (a), "=&r" (b)
 454                                   : "r" (dest), "r" (exch), "r" (comp)
 455                                   : "cc", "memory");
 456
 457         return a;
 458 }
 459
 460 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
 461 {
 462         gpointer a, b;
 463
 464         __asm__ __volatile__ (    "0:\n\t"
 465                                   "ldr %1, [%2]\n\t"
 466                                   "cmp %1, %4\n\t"
 467                                   "bne 1f\n\t"
 468                                   "swpeq %0, %3, [%2]\n\t"
 469                                   "cmp %0, %1\n\t"
 470                                   "swpne %3, %0, [%2]\n\t"
 471                                   "bne 0b\n\t"
 472                                   "1:"
 473                                   : "=&r" (a), "=&r" (b)
 474                                   : "r" (dest), "r" (exch), "r" (comp)
 475                                   : "cc", "memory");
 476
 477         return a;
 478 }
 479
 480 static inline gint32 InterlockedIncrement(volatile gint32 *dest)
 481 {
 482         int a, b, c;
 483
 484         __asm__ __volatile__ (  "0:\n\t"
 485                                 "ldr %0, [%3]\n\t"
 486                                 "add %1, %0, %4\n\t"
 487                                 "swp %2, %1, [%3]\n\t"
 488                                 "cmp %0, %2\n\t"
 489                                 "swpne %1, %2, [%3]\n\t"
 490                                 "bne 0b"
 491                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 492                                 : "r" (dest), "r" (1)
 493                                 : "cc", "memory");
 494
 495         return b;
 496 }
 497
 498 static inline gint32 InterlockedDecrement(volatile gint32 *dest)
 499 {
 500         int a, b, c;
 501
 502         __asm__ __volatile__ (  "0:\n\t"
 503                                 "ldr %0, [%3]\n\t"
 504                                 "add %1, %0, %4\n\t"
 505                                 "swp %2, %1, [%3]\n\t"
 506                                 "cmp %0, %2\n\t"
 507                                 "swpne %1, %2, [%3]\n\t"
 508                                 "bne 0b"
 509                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 510                                 : "r" (dest), "r" (-1)
 511                                 : "cc", "memory");
 512
 513         return b;
 514 }
 515
 516 static inline gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch)
 517 {
 518         int a;
 519
 520         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 521                                 : "=&r" (a)
 522                                 : "r" (dest), "r" (exch));
 523
 524         return a;
 525 }
 526
 527 static inline gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch)
 528 {
 529         gpointer a;
 530
 531         __asm__ __volatile__ (  "swp %0, %2, [%1]"
 532                                 : "=&r" (a)
 533                                 : "r" (dest), "r" (exch));
 534
 535         return a;
 536 }
 537
 538 static inline gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add)
 539 {
 540         int a, b, c;
 541
 542         __asm__ __volatile__ (  "0:\n\t"
 543                                 "ldr %0, [%3]\n\t"
 544                                 "add %1, %0, %4\n\t"
 545                                 "swp %2, %1, [%3]\n\t"
 546                                 "cmp %0, %2\n\t"
 547                                 "swpne %1, %2, [%3]\n\t"
 548                                 "bne 0b"
 549                                 : "=&r" (a), "=&r" (b), "=&r" (c)
 550                                 : "r" (dest), "r" (add)
 551                                 : "cc", "memory");
 552
 553         return a;
 554 }
 555
 556 #else
 557
 558 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
 559 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
 560 extern gint32 InterlockedIncrement(volatile gint32 *dest);
 561 extern gint32 InterlockedDecrement(volatile gint32 *dest);
 562 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
 563 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
 564 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
 565
 566 #if defined(__hpux) && !defined(__GNUC__)
 567 #define WAPI_ATOMIC_ASM
 568 #endif
 569
 570 #endif
 571
 572 #endif /* _WAPI_ATOMIC_H_ */