2 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1999-2003 by Hewlett-Packard Company. All rights reserved.
7 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
8 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
10 * Permission is hereby granted to use or copy this program
11 * for any purpose, provided the above notices are retained on all copies.
12 * Permission to modify the code and to distribute modified code is granted,
13 * provided the above notices are retained, and a notice that the code was
14 * modified is included with the above copyright notice.
18 #include "../read_ordered.h"
20 #include "../test_and_set_t_is_ao_t.h" /* Probably suboptimal */
22 /* NEC LE-IT: ARMv6 is the first architecture providing support for simple LL/SC
23 * A data memory barrier must be raised via CP15 command (see documentation).
25 * ARMv7 is compatible to ARMv6 but has a simpler command for issuing a
26 * memory barrier (DMB). Raising it via CP15 should still work as told me by the
27 * support engineers. If it turns out to be much quicker than we should implement
28 * custom code for ARMv7 using the asm { dmb } command.
30 * If only a single processor is used, we can define AO_UNIPROCESSOR
31 * and do not need to access CP15 for ensuring a DMB
34 /* NEC LE-IT: gcc has no way to easily check the arm architecture
35 * but defines only one of __ARM_ARCH_x__ to be true */
36 #if defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_7__)
38 #include "../standard_ao_double_t.h"
43 #ifndef AO_UNIPROCESSOR
44 /* issue an data memory barrier (keeps ordering of memory transactions */
45 /* before and after this operation) */
47 __asm__ __volatile__("mcr p15,0,%0,c7,c10,5" :"=&r"(dest) : : "memory");
51 #define AO_HAVE_nop_full
53 /* NEC LE-IT: AO_t load is simple reading */
55 AO_load(const volatile AO_t *addr)
57 /* Cast away the volatile for architectures like IA64 where */
58 /* volatile adds barrier semantics. */
59 return (*(const AO_t *)addr);
63 /* NEC LE-IT: atomic "store" - according to ARM documentation this is
64 * the only safe way to set variables also used in LL/SC environment.
65 * A direct write won't be recognized by the LL/SC construct on the _same_ CPU.
66 * Support engineers response for behaviour of ARMv6:
69 ===================================
72 -----------------------------------
76 -----------------------------------
80 -----------------------------------
82 * ARMv7 behaves similar, see documentation CortexA8 TRM, point 8.5
84 * HB: I think this is only a problem if interrupt handlers do not clear
85 * the reservation, as they almost certainly should. Probably change this back
88 AO_INLINE void AO_store(volatile AO_t *addr, AO_t value)
92 __asm__ __volatile__("@AO_store\n"
94 " strex %0, %3, [%2]\n"
97 : "=&r"(flag), "+m"(*addr)
98 : "r" (addr), "r"(value)
101 #define AO_HAVE_store
103 /* NEC LE-IT: replace the SWAP as recommended by ARM:
105 "Applies to: ARM11 Cores
106 Though the SWP instruction will still work with ARM V6 cores, it is
107 recommended to use the new V6 synchronization instructions. The SWP
108 instruction produces ‘locked’ read and write accesses which are atomic,
109 i.e. another operation cannot be done between these locked accesses which
110 ties up external bus (AHB,AXI) bandwidth and can increase worst case
111 interrupt latencies. LDREX,STREX are more flexible, other instructions can
112 be done between the LDREX and STREX accesses.
116 AO_test_and_set(volatile AO_TS_t *addr) {
121 __asm__ __volatile__("@AO_test_and_set\n"
122 "1: ldrex %0, [%3]\n"
123 " strex %1, %4, [%3]\n"
126 : "=&r"(oldval),"=&r"(flag), "+m"(*addr)
133 #define AO_HAVE_test_and_set
135 /* NEC LE-IT: fetch and add for ARMv6 */
137 AO_fetch_and_add(volatile AO_t *p, AO_t incr)
139 unsigned long flag,tmp;
142 __asm__ __volatile__("@AO_fetch_and_add\n"
143 "1: ldrex %0, [%5]\n" /* get original */
144 " add %2, %0, %4\n" /* sum up in incr */
145 " strex %1, %2, [%5]\n" /* store them */
148 : "=&r"(result),"=&r"(flag),"=&r"(tmp),"+m"(*p) /* 0..3 */
149 : "r"(incr), "r"(p) /* 4..5 */
155 #define AO_HAVE_fetch_and_add
157 /* NEC LE-IT: fetch and add1 for ARMv6 */
159 AO_fetch_and_add1(volatile AO_t *p)
161 unsigned long flag,tmp;
164 __asm__ __volatile__("@AO_fetch_and_add1\n"
165 "1: ldrex %0, [%4]\n" /* get original */
166 " add %1, %0, #1\n" /* increment */
167 " strex %2, %1, [%4]\n" /* store them */
170 : "=&r"(result), "=&r"(tmp), "=&r"(flag), "+m"(*p)
177 #define AO_HAVE_fetch_and_add1
179 /* NEC LE-IT: fetch and sub for ARMv6 */
181 AO_fetch_and_sub1(volatile AO_t *p)
183 unsigned long flag,tmp;
186 __asm__ __volatile__("@AO_fetch_and_sub1\n"
187 "1: ldrex %0, [%4]\n" /* get original */
188 " sub %1, %0, #1\n" /* decrement */
189 " strex %2, %1, [%4]\n" /* store them */
192 : "=&r"(result), "=&r"(tmp), "=&r"(flag), "+m"(*p)
199 #define AO_HAVE_fetch_and_sub1
201 /* NEC LE-IT: compare and swap */
202 /* Returns nonzero if the comparison succeeded. */
204 AO_compare_and_swap(volatile AO_t *addr,
205 AO_t old_val, AO_t new_val)
209 __asm__ __volatile__("@ AO_compare_and_swap\n"
210 "1: mov %0, #2\n" /* store a flag */
211 " ldrex %1, [%3]\n" /* get original */
212 " teq %1, %4\n" /* see if match */
213 " strexeq %0, %5, [%3]\n" /* store new one if matched */
215 " beq 1b\n" /* if update failed, repeat */
216 : "=&r"(result), "=&r"(tmp), "+m"(*addr)
217 : "r"(addr), "r"(old_val), "r"(new_val)
220 return !(result&2); /* if succeded, return 1, else 0 */
222 #define AO_HAVE_compare_and_swap
225 AO_compare_double_and_swap_double(volatile AO_double_t *addr,
226 AO_t old_val1, AO_t old_val2,
227 AO_t new_val1, AO_t new_val2)
229 double_ptr_storage old_val = ((double_ptr_storage)old_val2 << 32) | old_val1;
230 double_ptr_storage new_val = ((double_ptr_storage)new_val2 << 32) | new_val1;
232 double_ptr_storage tmp;
236 __asm__ __volatile__("@ AO_compare_and_swap_double\n"
237 " ldrexd %0, [%1]\n" /* get original to r1&r2*/
241 if(tmp != old_val) return false;
242 __asm__ __volatile__(
243 " strexd %0, %2, [%3]\n" /* store new one if matched */
244 : "=&r"(result),"+m"(*addr)
245 : "r"(new_val), "r"(addr)
247 if(!result) return true;
251 #define AO_HAVE_compare_double_and_swap_double
254 /* pre ARMv6 architecures ... */
256 /* I found a slide set that, if I read it correctly, claims that */
257 /* Loads followed by either a Load or Store are ordered, but nothing */
259 /* It appears that SWP is the only simple memory barrier. */
260 #include "../all_atomic_load_store.h"
262 AO_INLINE AO_TS_VAL_t
263 AO_test_and_set_full(volatile AO_TS_t *addr) {
265 /* SWP on ARM is very similar to XCHG on x86. */
266 /* The first operand is the result, the second the value */
267 /* to be stored. Both registers must be different from addr. */
268 /* Make the address operand an early clobber output so it */
269 /* doesn't overlap with the other operands. The early clobber*/
270 /* on oldval is necessary to prevent the compiler allocating */
271 /* them to the same register if they are both unused. */
272 __asm__ __volatile__("swp %0, %2, [%3]"
273 : "=&r"(oldval), "=&r"(addr)
279 #define AO_HAVE_test_and_set_full
281 #endif // __ARM_ARCH_x