[arm64] Add sequential memory constraint to swap, cas and add

Previously we were wrongly trying to achieve this by doing a load-acq followed by a store-rel. Assuming we have memory access on address A (before) and C (after) while the atomic operation happens on address B, the order of the instructions would look like this :

	[A], [B]load, load-acq, store-rel, [B]store, [C]

In this case a reordering of memory accesses like the following is possible, which clearly breaks the desired constraints :

	[B]load, load-acq, [C], [A], store-rel, [B]store

In order to provide the sequential memory constraint we could emit the barriers like this instead :

	[A], membar, [B]load, [B]store, membar, [C]

We can additionally save an instruction and emit the barriers like this instead (this is also what gcc does) :

	[A], [B]load, store-rel, [B]store, membar, [C]

In this case we only need to worry about the relation between memory accesses on A and the loading of B. We need to consider what happens if B is loaded before the access on A is fulfilled. This only matters if we load an older value for B (rather than the one visible by other threads) before A. In that case the erroneous store will fail due to using exclusive stores. At the retry the value read from B will be the right one.

diff --git a/mono/mini/mini-arm64.c b/mono/mini/mini-arm64.c
index 0a499a6375115110adc23d3e200ede9bcc21d3cd..688594d96602b50770485925c92a521726edeea7 100644 (file)
--- a/mono/mini/mini-arm64.c
+++ b/mono/mini/mini-arm64.c
@@ -3631,11 +3631,12 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        guint8 *buf [16];
 
                        buf [0] = code;
-                       arm_ldaxrw (code, ARMREG_IP0, sreg1);
+                       arm_ldxrw (code, ARMREG_IP0, sreg1);
                        arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
                        arm_stlxrw (code, ARMREG_IP1, ARMREG_IP0, sreg1);
                        arm_cbnzw (code, ARMREG_IP1, buf [0]);
 
+                       arm_dmb (code, 0);
                        arm_movx (code, dreg, ARMREG_IP0);
                        break;
                }
@@ -3643,11 +3644,12 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        guint8 *buf [16];
 
                        buf [0] = code;
-                       arm_ldaxrx (code, ARMREG_IP0, sreg1);
+                       arm_ldxrx (code, ARMREG_IP0, sreg1);
                        arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
                        arm_stlxrx (code, ARMREG_IP1, ARMREG_IP0, sreg1);
                        arm_cbnzx (code, ARMREG_IP1, buf [0]);
 
+                       arm_dmb (code, 0);
                        arm_movx (code, dreg, ARMREG_IP0);
                        break;
                }
@@ -3655,10 +3657,11 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        guint8 *buf [16];
 
                        buf [0] = code;
-                       arm_ldaxrw (code, ARMREG_IP0, sreg1);
+                       arm_ldxrw (code, ARMREG_IP0, sreg1);
                        arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
                        arm_cbnzw (code, ARMREG_IP1, buf [0]);
 
+                       arm_dmb (code, 0);
                        arm_movx (code, dreg, ARMREG_IP0);
                        break;
                }
@@ -3666,10 +3669,11 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        guint8 *buf [16];
 
                        buf [0] = code;
-                       arm_ldaxrx (code, ARMREG_IP0, sreg1);
+                       arm_ldxrx (code, ARMREG_IP0, sreg1);
                        arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
                        arm_cbnzw (code, ARMREG_IP1, buf [0]);
 
+                       arm_dmb (code, 0);
                        arm_movx (code, dreg, ARMREG_IP0);
                        break;
                }
@@ -3678,7 +3682,7 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
 
                        /* sreg2 is the value, sreg3 is the comparand */
                        buf [0] = code;
-                       arm_ldaxrw (code, ARMREG_IP0, sreg1);
+                       arm_ldxrw (code, ARMREG_IP0, sreg1);
                        arm_cmpw (code, ARMREG_IP0, ins->sreg3);
                        buf [1] = code;
                        arm_bcc (code, ARMCOND_NE, 0);
@@ -3686,6 +3690,7 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        arm_cbnzw (code, ARMREG_IP1, buf [0]);
                        arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
 
+                       arm_dmb (code, 0);
                        arm_movx (code, dreg, ARMREG_IP0);
                        break;
                }
@@ -3693,7 +3698,7 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        guint8 *buf [16];
 
                        buf [0] = code;
-                       arm_ldaxrx (code, ARMREG_IP0, sreg1);
+                       arm_ldxrx (code, ARMREG_IP0, sreg1);
                        arm_cmpx (code, ARMREG_IP0, ins->sreg3);
                        buf [1] = code;
                        arm_bcc (code, ARMCOND_NE, 0);
@@ -3701,6 +3706,7 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
                        arm_cbnzw (code, ARMREG_IP1, buf [0]);
                        arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
 
+                       arm_dmb (code, 0);
                        arm_movx (code, dreg, ARMREG_IP0);
                        break;
                }