2 * atomic.h: Atomic operations
5 * Dick Porter (dick@ximian.com)
7 * (C) 2002 Ximian, Inc.
8 * Copyright 2012 Xamarin Inc
11 #ifndef _WAPI_ATOMIC_H_
12 #define _WAPI_ATOMIC_H_
17 #ifdef ENABLE_EXTENSION_MODULE
18 #include "../../../mono-extensions/mono/utils/atomic.h"
21 /* On Windows, we always use the functions provided by the Windows API. */
22 #if defined(__WIN32__) || defined(_WIN32)
24 #ifndef WIN32_LEAN_AND_MEAN
25 #define WIN32_LEAN_AND_MEAN
28 #include <mono/utils/mono-membar.h>
30 /* mingw is missing InterlockedCompareExchange64 () from winbase.h */
31 #if HAVE_DECL_INTERLOCKEDCOMPAREEXCHANGE64==0
32 static inline gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
34 return __sync_val_compare_and_swap (dest, comp, exch);
38 /* mingw is missing InterlockedExchange64 () from winbase.h */
39 #if HAVE_DECL_INTERLOCKEDEXCHANGE64==0
40 static inline gint64 InterlockedExchange64(volatile gint64 *val, gint64 new_val)
45 } while (InterlockedCompareExchange64 (val, new_val, old_val) != old_val);
50 /* mingw is missing InterlockedIncrement64 () from winbase.h */
51 #if HAVE_DECL_INTERLOCKEDINCREMENT64==0
52 static inline gint64 InterlockedIncrement64(volatile gint64 *val)
54 return __sync_add_and_fetch (val, 1);
58 /* mingw is missing InterlockedDecrement64 () from winbase.h */
59 #if HAVE_DECL_INTERLOCKEDDECREMENT64==0
60 static inline gint64 InterlockedDecrement64(volatile gint64 *val)
62 return __sync_sub_and_fetch (val, 1);
66 /* mingw is missing InterlockedAdd () from winbase.h */
67 #if HAVE_DECL_INTERLOCKEDADD==0
68 static inline gint32 InterlockedAdd(volatile gint32 *dest, gint32 add)
70 return __sync_add_and_fetch (dest, add);
74 /* mingw is missing InterlockedAdd64 () from winbase.h */
75 #if HAVE_DECL_INTERLOCKEDADD64==0
76 static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
78 return __sync_add_and_fetch (dest, add);
82 #if defined(_MSC_VER) && !defined(InterlockedAdd)
83 /* MSVC before 2013 only defines InterlockedAdd* for the Itanium architecture */
84 static inline gint32 InterlockedAdd(volatile gint32 *dest, gint32 add)
86 return InterlockedExchangeAdd (dest, add) + add;
90 #if defined(_MSC_VER) && !defined(InterlockedAdd64)
91 #if defined(InterlockedExchangeAdd64)
92 /* This may be defined only on amd64 */
93 static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
95 return InterlockedExchangeAdd64 (dest, add) + add;
98 static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
104 } while (prev_value != InterlockedCompareExchange64(dest, prev_value + add, prev_value));
106 return prev_value + add;
112 #define TO_INTERLOCKED_ARGP(ptr) ((volatile LONG*)(ptr))
114 #define TO_INTERLOCKED_ARGP(ptr) (ptr)
117 /* And now for some dirty hacks... The Windows API doesn't
118 * provide any useful primitives for this (other than getting
119 * into architecture-specific madness), so use CAS. */
121 static inline gint32 InterlockedRead(volatile gint32 *src)
123 return InterlockedCompareExchange (TO_INTERLOCKED_ARGP (src), 0, 0);
126 static inline gint64 InterlockedRead64(volatile gint64 *src)
128 return InterlockedCompareExchange64 (src, 0, 0);
131 static inline gpointer InterlockedReadPointer(volatile gpointer *src)
133 return InterlockedCompareExchangePointer (src, NULL, NULL);
136 static inline void InterlockedWrite(volatile gint32 *dst, gint32 val)
138 InterlockedExchange (TO_INTERLOCKED_ARGP (dst), val);
141 static inline void InterlockedWrite64(volatile gint64 *dst, gint64 val)
143 InterlockedExchange64 (dst, val);
146 static inline void InterlockedWritePointer(volatile gpointer *dst, gpointer val)
148 InterlockedExchangePointer (dst, val);
151 /* We can't even use CAS for these, so write them out
152 * explicitly according to x86(_64) semantics... */
154 static inline gint8 InterlockedRead8(volatile gint8 *src)
159 static inline gint16 InterlockedRead16(volatile gint16 *src)
164 static inline void InterlockedWrite8(volatile gint8 *dst, gint8 val)
167 mono_memory_barrier ();
170 static inline void InterlockedWrite16(volatile gint16 *dst, gint16 val)
173 mono_memory_barrier ();
176 /* Prefer GCC atomic ops if the target supports it (see configure.ac). */
177 #elif defined(USE_GCC_ATOMIC_OPS)
179 static inline gint32 InterlockedCompareExchange(volatile gint32 *dest,
180 gint32 exch, gint32 comp)
182 return __sync_val_compare_and_swap (dest, comp, exch);
185 static inline gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp)
187 return __sync_val_compare_and_swap (dest, comp, exch);
190 static inline gint32 InterlockedAdd(volatile gint32 *dest, gint32 add)
192 return __sync_add_and_fetch (dest, add);
195 static inline gint32 InterlockedIncrement(volatile gint32 *val)
197 return __sync_add_and_fetch (val, 1);
200 static inline gint32 InterlockedDecrement(volatile gint32 *val)
202 return __sync_sub_and_fetch (val, 1);
205 static inline gint32 InterlockedExchange(volatile gint32 *val, gint32 new_val)
210 } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
214 static inline gpointer InterlockedExchangePointer(volatile gpointer *val,
220 } while (__sync_val_compare_and_swap (val, old_val, new_val) != old_val);
224 static inline gint32 InterlockedExchangeAdd(volatile gint32 *val, gint32 add)
226 return __sync_fetch_and_add (val, add);
229 static inline gint8 InterlockedRead8(volatile gint8 *src)
231 /* Kind of a hack, but GCC doesn't give us anything better, and it's
232 * certainly not as bad as using a CAS loop. */
233 return __sync_fetch_and_add (src, 0);
236 static inline gint16 InterlockedRead16(volatile gint16 *src)
238 return __sync_fetch_and_add (src, 0);
241 static inline gint32 InterlockedRead(volatile gint32 *src)
243 return __sync_fetch_and_add (src, 0);
246 static inline void InterlockedWrite8(volatile gint8 *dst, gint8 val)
248 /* Nothing useful from GCC at all, so fall back to CAS. */
252 } while (__sync_val_compare_and_swap (dst, old_val, val) != old_val);
255 static inline void InterlockedWrite16(volatile gint16 *dst, gint16 val)
260 } while (__sync_val_compare_and_swap (dst, old_val, val) != old_val);
263 static inline void InterlockedWrite(volatile gint32 *dst, gint32 val)
265 /* Nothing useful from GCC at all, so fall back to CAS. */
269 } while (__sync_val_compare_and_swap (dst, old_val, val) != old_val);
272 #if defined (TARGET_OSX) || defined (__arm__) || (defined (__mips__) && !defined (__mips64)) || (defined (__powerpc__) && !defined (__powerpc64__)) || (defined (__sparc__) && !defined (__arch64__))
273 #define BROKEN_64BIT_ATOMICS_INTRINSIC 1
276 #if !defined (BROKEN_64BIT_ATOMICS_INTRINSIC)
278 static inline gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp)
280 return __sync_val_compare_and_swap (dest, comp, exch);
283 static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
285 return __sync_add_and_fetch (dest, add);
288 static inline gint64 InterlockedIncrement64(volatile gint64 *val)
290 return __sync_add_and_fetch (val, 1);
293 static inline gint64 InterlockedDecrement64(volatile gint64 *val)
295 return __sync_sub_and_fetch (val, 1);
298 static inline gint64 InterlockedExchangeAdd64(volatile gint64 *val, gint64 add)
300 return __sync_fetch_and_add (val, add);
303 static inline gint64 InterlockedRead64(volatile gint64 *src)
305 /* Kind of a hack, but GCC doesn't give us anything better. */
306 return __sync_fetch_and_add (src, 0);
311 /* Implement 64-bit cmpxchg by hand or emulate it. */
312 extern gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp);
314 /* Implement all other 64-bit atomics in terms of a specialized CAS
315 * in this case, since chances are that the other 64-bit atomic
316 * intrinsics are broken too.
319 static inline gint64 InterlockedExchangeAdd64(volatile gint64 *dest, gint64 add)
324 } while (InterlockedCompareExchange64 (dest, old_val + add, old_val) != old_val);
328 static inline gint64 InterlockedIncrement64(volatile gint64 *val)
334 } while (InterlockedCompareExchange64 (val, set, get) != get);
338 static inline gint64 InterlockedDecrement64(volatile gint64 *val)
344 } while (InterlockedCompareExchange64 (val, set, get) != get);
348 static inline gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add)
354 } while (InterlockedCompareExchange64 (dest, set, get) != get);
358 static inline gint64 InterlockedRead64(volatile gint64 *src)
360 return InterlockedCompareExchange64 (src, 0, 0);
365 static inline gpointer InterlockedReadPointer(volatile gpointer *src)
367 return InterlockedCompareExchangePointer (src, NULL, NULL);
370 static inline void InterlockedWritePointer(volatile gpointer *dst, gpointer val)
372 InterlockedExchangePointer (dst, val);
375 /* We always implement this in terms of a 64-bit cmpxchg since
376 * GCC doesn't have an intrisic to model it anyway. */
377 static inline gint64 InterlockedExchange64(volatile gint64 *val, gint64 new_val)
382 } while (InterlockedCompareExchange64 (val, new_val, old_val) != old_val);
386 static inline void InterlockedWrite64(volatile gint64 *dst, gint64 val)
388 /* Nothing useful from GCC at all, so fall back to CAS. */
389 InterlockedExchange64 (dst, val);
392 #elif defined(__ia64__)
394 #ifdef __INTEL_COMPILER
395 #include <ia64intrin.h>
398 static inline gint32 InterlockedCompareExchange(gint32 volatile *dest,
399 gint32 exch, gint32 comp)
404 #ifdef __INTEL_COMPILER
405 old = _InterlockedCompareExchange (dest, exch, comp);
407 /* cmpxchg4 zero extends the value read from memory */
408 real_comp = (guint64)(guint32)comp;
409 asm volatile ("mov ar.ccv = %2 ;;\n\t"
410 "cmpxchg4.acq %0 = [%1], %3, ar.ccv\n\t"
411 : "=r" (old) : "r" (dest), "r" (real_comp), "r" (exch));
417 static inline gpointer InterlockedCompareExchangePointer(gpointer volatile *dest,
418 gpointer exch, gpointer comp)
422 #ifdef __INTEL_COMPILER
423 old = _InterlockedCompareExchangePointer (dest, exch, comp);
425 asm volatile ("mov ar.ccv = %2 ;;\n\t"
426 "cmpxchg8.acq %0 = [%1], %3, ar.ccv\n\t"
427 : "=r" (old) : "r" (dest), "r" (comp), "r" (exch));
433 static inline gint32 InterlockedIncrement(gint32 volatile *val)
435 #ifdef __INTEL_COMPILER
436 return _InterlockedIncrement (val);
442 } while (InterlockedCompareExchange (val, old + 1, old) != old);
448 static inline gint32 InterlockedDecrement(gint32 volatile *val)
450 #ifdef __INTEL_COMPILER
451 return _InterlockedDecrement (val);
457 } while (InterlockedCompareExchange (val, old - 1, old) != old);
463 static inline gint32 InterlockedExchange(gint32 volatile *dest, gint32 new_val)
465 #ifdef __INTEL_COMPILER
466 return _InterlockedExchange (dest, new_val);
472 } while (InterlockedCompareExchange (dest, new_val, res) != res);
478 static inline gpointer InterlockedExchangePointer(gpointer volatile *dest, gpointer new_val)
480 #ifdef __INTEL_COMPILER
481 return (gpointer)_InterlockedExchange64 ((gint64*)dest, (gint64)new_val);
487 } while (InterlockedCompareExchangePointer (dest, new_val, res) != res);
493 static inline gint32 InterlockedExchangeAdd(gint32 volatile *val, gint32 add)
497 #ifdef __INTEL_COMPILER
498 old = _InterlockedExchangeAdd (val, add);
502 } while (InterlockedCompareExchange (val, old + add, old) != old);
510 #define WAPI_NO_ATOMIC_ASM
512 extern gint32 InterlockedCompareExchange(volatile gint32 *dest, gint32 exch, gint32 comp);
513 extern gint64 InterlockedCompareExchange64(volatile gint64 *dest, gint64 exch, gint64 comp);
514 extern gpointer InterlockedCompareExchangePointer(volatile gpointer *dest, gpointer exch, gpointer comp);
515 extern gint32 InterlockedAdd(volatile gint32 *dest, gint32 add);
516 extern gint64 InterlockedAdd64(volatile gint64 *dest, gint64 add);
517 extern gint32 InterlockedIncrement(volatile gint32 *dest);
518 extern gint64 InterlockedIncrement64(volatile gint64 *dest);
519 extern gint32 InterlockedDecrement(volatile gint32 *dest);
520 extern gint64 InterlockedDecrement64(volatile gint64 *dest);
521 extern gint32 InterlockedExchange(volatile gint32 *dest, gint32 exch);
522 extern gint64 InterlockedExchange64(volatile gint64 *dest, gint64 exch);
523 extern gpointer InterlockedExchangePointer(volatile gpointer *dest, gpointer exch);
524 extern gint32 InterlockedExchangeAdd(volatile gint32 *dest, gint32 add);
525 extern gint64 InterlockedExchangeAdd64(volatile gint64 *dest, gint64 add);
526 extern gint8 InterlockedRead8(volatile gint8 *src);
527 extern gint16 InterlockedRead16(volatile gint16 *src);
528 extern gint32 InterlockedRead(volatile gint32 *src);
529 extern gint64 InterlockedRead64(volatile gint64 *src);
530 extern gpointer InterlockedReadPointer(volatile gpointer *src);
531 extern void InterlockedWrite8(volatile gint8 *dst, gint8 val);
532 extern void InterlockedWrite16(volatile gint16 *dst, gint16 val);
533 extern void InterlockedWrite(volatile gint32 *dst, gint32 val);
534 extern void InterlockedWrite64(volatile gint64 *dst, gint64 val);
535 extern void InterlockedWritePointer(volatile gpointer *dst, gpointer val);
539 #endif /* _WAPI_ATOMIC_H_ */