2 * mini-arm64.c: ARM64 backend for the Mono code generator
4 * Copyright 2013 Xamarin, Inc (http://www.xamarin.com)
9 * Paolo Molaro (lupus@ximian.com)
10 * Dietmar Maurer (dietmar@ximian.com)
12 * (C) 2003 Ximian, Inc.
13 * Copyright 2003-2011 Novell, Inc (http://www.novell.com)
14 * Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
15 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
19 #include "cpu-arm64.h"
22 #include <mono/arch/arm64/arm64-codegen.h>
23 #include <mono/utils/mono-mmap.h>
24 #include <mono/utils/mono-memory-model.h>
25 #include <mono/metadata/abi-details.h>
30 * - ARM(R) Architecture Reference Manual, ARMv8, for ARMv8-A architecture profile (DDI0487A_a_armv8_arm.pdf)
31 * - Procedure Call Standard for the ARM 64-bit Architecture (AArch64) (IHI0055B_aapcs64.pdf)
32 * - ELF for the ARM 64-bit Architecture (IHI0056B_aaelf64.pdf)
35 * - ip0/ip1/lr are used as temporary registers
36 * - r27 is used as the rgctx/imt register
37 * - r28 is used to access arguments passed on the stack
38 * - d15/d16 are used as fp temporary registers
41 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
43 #define FP_TEMP_REG ARMREG_D16
44 #define FP_TEMP_REG2 ARMREG_D17
46 #define THUNK_SIZE (4 * 4)
48 /* The single step trampoline */
49 static gpointer ss_trampoline;
51 /* The breakpoint trampoline */
52 static gpointer bp_trampoline;
54 static gboolean ios_abi;
56 static __attribute__((warn_unused_result)) guint8* emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset);
59 mono_arch_regname (int reg)
61 static const char * rnames[] = {
62 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9",
63 "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17", "r18", "r19",
64 "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "fp",
67 if (reg >= 0 && reg < 32)
73 mono_arch_fregname (int reg)
75 static const char * rnames[] = {
76 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9",
77 "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19",
78 "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29",
81 if (reg >= 0 && reg < 32)
87 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
93 #define MAX_ARCH_DELEGATE_PARAMS 7
96 get_delegate_invoke_impl (gboolean has_target, gboolean param_count, guint32 *code_size)
101 start = code = mono_global_codeman_reserve (12);
103 /* Replace the this argument with the target */
104 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
105 arm_ldrx (code, ARMREG_R0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, target));
106 arm_brx (code, ARMREG_IP0);
108 g_assert ((code - start) <= 12);
110 mono_arch_flush_icache (start, 12);
114 size = 8 + param_count * 4;
115 start = code = mono_global_codeman_reserve (size);
117 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
118 /* slide down the arguments */
119 for (i = 0; i < param_count; ++i)
120 arm_movx (code, i, i + 1);
121 arm_brx (code, ARMREG_IP0);
123 g_assert ((code - start) <= size);
125 mono_arch_flush_icache (start, size);
129 *code_size = code - start;
135 * mono_arch_get_delegate_invoke_impls:
137 * Return a list of MonoAotTrampInfo structures for the delegate invoke impl
141 mono_arch_get_delegate_invoke_impls (void)
149 code = get_delegate_invoke_impl (TRUE, 0, &code_len);
150 res = g_slist_prepend (res, mono_tramp_info_create ("delegate_invoke_impl_has_target", code, code_len, NULL, NULL));
152 for (i = 0; i <= MAX_ARCH_DELEGATE_PARAMS; ++i) {
153 code = get_delegate_invoke_impl (FALSE, i, &code_len);
154 tramp_name = g_strdup_printf ("delegate_invoke_impl_target_%d", i);
155 res = g_slist_prepend (res, mono_tramp_info_create (tramp_name, code, code_len, NULL, NULL));
163 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
165 guint8 *code, *start;
168 * vtypes are returned in registers, or using the dedicated r8 register, so
169 * they can be supported by delegate invokes.
173 static guint8* cached = NULL;
179 start = mono_aot_get_trampoline ("delegate_invoke_impl_has_target");
181 start = get_delegate_invoke_impl (TRUE, 0, NULL);
182 mono_memory_barrier ();
186 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
189 if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
191 for (i = 0; i < sig->param_count; ++i)
192 if (!mono_is_regsize_var (sig->params [i]))
195 code = cache [sig->param_count];
200 char *name = g_strdup_printf ("delegate_invoke_impl_target_%d", sig->param_count);
201 start = mono_aot_get_trampoline (name);
204 start = get_delegate_invoke_impl (FALSE, sig->param_count, NULL);
206 mono_memory_barrier ();
207 cache [sig->param_count] = start;
215 mono_arch_get_delegate_virtual_invoke_impl (MonoMethodSignature *sig, MonoMethod *method, int offset, gboolean load_imt_reg)
221 mono_arch_get_this_arg_from_call (mgreg_t *regs, guint8 *code)
223 return (gpointer)regs [ARMREG_R0];
227 mono_arch_cpu_init (void)
232 mono_arch_init (void)
234 mono_aot_register_jit_icall ("mono_arm_throw_exception", mono_arm_throw_exception);
235 mono_aot_register_jit_icall ("mono_arm_resume_unwind", mono_arm_resume_unwind);
238 bp_trampoline = mini_get_breakpoint_trampoline ();
240 mono_arm_gsharedvt_init ();
242 #if defined(TARGET_IOS)
248 mono_arch_cleanup (void)
253 mono_arch_cpu_optimizations (guint32 *exclude_mask)
260 mono_arch_cpu_enumerate_simd_versions (void)
266 mono_arch_register_lowlevel_calls (void)
271 mono_arch_finish_init (void)
275 /* The maximum length is 2 instructions */
277 emit_imm (guint8 *code, int dreg, int imm)
279 // FIXME: Optimize this
282 arm_movnx (code, dreg, (~limm) & 0xffff, 0);
283 arm_movkx (code, dreg, (limm >> 16) & 0xffff, 16);
285 arm_movzx (code, dreg, imm & 0xffff, 0);
287 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
293 /* The maximum length is 4 instructions */
295 emit_imm64 (guint8 *code, int dreg, guint64 imm)
297 // FIXME: Optimize this
298 arm_movzx (code, dreg, imm & 0xffff, 0);
299 if ((imm >> 16) & 0xffff)
300 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
301 if ((imm >> 32) & 0xffff)
302 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
303 if ((imm >> 48) & 0xffff)
304 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
310 mono_arm_emit_imm64 (guint8 *code, int dreg, gint64 imm)
312 return emit_imm64 (code, dreg, imm);
318 * Emit a patchable code sequence for constructing a 64 bit immediate.
321 emit_imm64_template (guint8 *code, int dreg)
323 arm_movzx (code, dreg, 0, 0);
324 arm_movkx (code, dreg, 0, 16);
325 arm_movkx (code, dreg, 0, 32);
326 arm_movkx (code, dreg, 0, 48);
331 static inline __attribute__((warn_unused_result)) guint8*
332 emit_addw_imm (guint8 *code, int dreg, int sreg, int imm)
334 if (!arm_is_arith_imm (imm)) {
335 code = emit_imm (code, ARMREG_LR, imm);
336 arm_addw (code, dreg, sreg, ARMREG_LR);
338 arm_addw_imm (code, dreg, sreg, imm);
343 static inline __attribute__((warn_unused_result)) guint8*
344 emit_addx_imm (guint8 *code, int dreg, int sreg, int imm)
346 if (!arm_is_arith_imm (imm)) {
347 code = emit_imm (code, ARMREG_LR, imm);
348 arm_addx (code, dreg, sreg, ARMREG_LR);
350 arm_addx_imm (code, dreg, sreg, imm);
355 static inline __attribute__((warn_unused_result)) guint8*
356 emit_subw_imm (guint8 *code, int dreg, int sreg, int imm)
358 if (!arm_is_arith_imm (imm)) {
359 code = emit_imm (code, ARMREG_LR, imm);
360 arm_subw (code, dreg, sreg, ARMREG_LR);
362 arm_subw_imm (code, dreg, sreg, imm);
367 static inline __attribute__((warn_unused_result)) guint8*
368 emit_subx_imm (guint8 *code, int dreg, int sreg, int imm)
370 if (!arm_is_arith_imm (imm)) {
371 code = emit_imm (code, ARMREG_LR, imm);
372 arm_subx (code, dreg, sreg, ARMREG_LR);
374 arm_subx_imm (code, dreg, sreg, imm);
379 /* Emit sp+=imm. Clobbers ip0/ip1 */
380 static inline __attribute__((warn_unused_result)) guint8*
381 emit_addx_sp_imm (guint8 *code, int imm)
383 code = emit_imm (code, ARMREG_IP0, imm);
384 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
385 arm_addx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
386 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
390 /* Emit sp-=imm. Clobbers ip0/ip1 */
391 static inline __attribute__((warn_unused_result)) guint8*
392 emit_subx_sp_imm (guint8 *code, int imm)
394 code = emit_imm (code, ARMREG_IP0, imm);
395 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
396 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
397 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
401 static inline __attribute__((warn_unused_result)) guint8*
402 emit_andw_imm (guint8 *code, int dreg, int sreg, int imm)
405 code = emit_imm (code, ARMREG_LR, imm);
406 arm_andw (code, dreg, sreg, ARMREG_LR);
411 static inline __attribute__((warn_unused_result)) guint8*
412 emit_andx_imm (guint8 *code, int dreg, int sreg, int imm)
415 code = emit_imm (code, ARMREG_LR, imm);
416 arm_andx (code, dreg, sreg, ARMREG_LR);
421 static inline __attribute__((warn_unused_result)) guint8*
422 emit_orrw_imm (guint8 *code, int dreg, int sreg, int imm)
425 code = emit_imm (code, ARMREG_LR, imm);
426 arm_orrw (code, dreg, sreg, ARMREG_LR);
431 static inline __attribute__((warn_unused_result)) guint8*
432 emit_orrx_imm (guint8 *code, int dreg, int sreg, int imm)
435 code = emit_imm (code, ARMREG_LR, imm);
436 arm_orrx (code, dreg, sreg, ARMREG_LR);
441 static inline __attribute__((warn_unused_result)) guint8*
442 emit_eorw_imm (guint8 *code, int dreg, int sreg, int imm)
445 code = emit_imm (code, ARMREG_LR, imm);
446 arm_eorw (code, dreg, sreg, ARMREG_LR);
451 static inline __attribute__((warn_unused_result)) guint8*
452 emit_eorx_imm (guint8 *code, int dreg, int sreg, int imm)
455 code = emit_imm (code, ARMREG_LR, imm);
456 arm_eorx (code, dreg, sreg, ARMREG_LR);
461 static inline __attribute__((warn_unused_result)) guint8*
462 emit_cmpw_imm (guint8 *code, int sreg, int imm)
465 arm_cmpw (code, sreg, ARMREG_RZR);
468 code = emit_imm (code, ARMREG_LR, imm);
469 arm_cmpw (code, sreg, ARMREG_LR);
475 static inline __attribute__((warn_unused_result)) guint8*
476 emit_cmpx_imm (guint8 *code, int sreg, int imm)
479 arm_cmpx (code, sreg, ARMREG_RZR);
482 code = emit_imm (code, ARMREG_LR, imm);
483 arm_cmpx (code, sreg, ARMREG_LR);
489 static inline __attribute__((warn_unused_result)) guint8*
490 emit_strb (guint8 *code, int rt, int rn, int imm)
492 if (arm_is_strb_imm (imm)) {
493 arm_strb (code, rt, rn, imm);
495 g_assert (rt != ARMREG_IP0);
496 g_assert (rn != ARMREG_IP0);
497 code = emit_imm (code, ARMREG_IP0, imm);
498 arm_strb_reg (code, rt, rn, ARMREG_IP0);
503 static inline __attribute__((warn_unused_result)) guint8*
504 emit_strh (guint8 *code, int rt, int rn, int imm)
506 if (arm_is_strh_imm (imm)) {
507 arm_strh (code, rt, rn, imm);
509 g_assert (rt != ARMREG_IP0);
510 g_assert (rn != ARMREG_IP0);
511 code = emit_imm (code, ARMREG_IP0, imm);
512 arm_strh_reg (code, rt, rn, ARMREG_IP0);
517 static inline __attribute__((warn_unused_result)) guint8*
518 emit_strw (guint8 *code, int rt, int rn, int imm)
520 if (arm_is_strw_imm (imm)) {
521 arm_strw (code, rt, rn, imm);
523 g_assert (rt != ARMREG_IP0);
524 g_assert (rn != ARMREG_IP0);
525 code = emit_imm (code, ARMREG_IP0, imm);
526 arm_strw_reg (code, rt, rn, ARMREG_IP0);
531 static inline __attribute__((warn_unused_result)) guint8*
532 emit_strfpw (guint8 *code, int rt, int rn, int imm)
534 if (arm_is_strw_imm (imm)) {
535 arm_strfpw (code, rt, rn, imm);
537 g_assert (rn != ARMREG_IP0);
538 code = emit_imm (code, ARMREG_IP0, imm);
539 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
540 arm_strfpw (code, rt, ARMREG_IP0, 0);
545 static inline __attribute__((warn_unused_result)) guint8*
546 emit_strfpx (guint8 *code, int rt, int rn, int imm)
548 if (arm_is_strx_imm (imm)) {
549 arm_strfpx (code, rt, rn, imm);
551 g_assert (rn != ARMREG_IP0);
552 code = emit_imm (code, ARMREG_IP0, imm);
553 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
554 arm_strfpx (code, rt, ARMREG_IP0, 0);
559 static inline __attribute__((warn_unused_result)) guint8*
560 emit_strx (guint8 *code, int rt, int rn, int imm)
562 if (arm_is_strx_imm (imm)) {
563 arm_strx (code, rt, rn, imm);
565 g_assert (rt != ARMREG_IP0);
566 g_assert (rn != ARMREG_IP0);
567 code = emit_imm (code, ARMREG_IP0, imm);
568 arm_strx_reg (code, rt, rn, ARMREG_IP0);
573 static inline __attribute__((warn_unused_result)) guint8*
574 emit_ldrb (guint8 *code, int rt, int rn, int imm)
576 if (arm_is_pimm12_scaled (imm, 1)) {
577 arm_ldrb (code, rt, rn, imm);
579 g_assert (rt != ARMREG_IP0);
580 g_assert (rn != ARMREG_IP0);
581 code = emit_imm (code, ARMREG_IP0, imm);
582 arm_ldrb_reg (code, rt, rn, ARMREG_IP0);
587 static inline __attribute__((warn_unused_result)) guint8*
588 emit_ldrsbx (guint8 *code, int rt, int rn, int imm)
590 if (arm_is_pimm12_scaled (imm, 1)) {
591 arm_ldrsbx (code, rt, rn, imm);
593 g_assert (rt != ARMREG_IP0);
594 g_assert (rn != ARMREG_IP0);
595 code = emit_imm (code, ARMREG_IP0, imm);
596 arm_ldrsbx_reg (code, rt, rn, ARMREG_IP0);
601 static inline __attribute__((warn_unused_result)) guint8*
602 emit_ldrh (guint8 *code, int rt, int rn, int imm)
604 if (arm_is_pimm12_scaled (imm, 2)) {
605 arm_ldrh (code, rt, rn, imm);
607 g_assert (rt != ARMREG_IP0);
608 g_assert (rn != ARMREG_IP0);
609 code = emit_imm (code, ARMREG_IP0, imm);
610 arm_ldrh_reg (code, rt, rn, ARMREG_IP0);
615 static inline __attribute__((warn_unused_result)) guint8*
616 emit_ldrshx (guint8 *code, int rt, int rn, int imm)
618 if (arm_is_pimm12_scaled (imm, 2)) {
619 arm_ldrshx (code, rt, rn, imm);
621 g_assert (rt != ARMREG_IP0);
622 g_assert (rn != ARMREG_IP0);
623 code = emit_imm (code, ARMREG_IP0, imm);
624 arm_ldrshx_reg (code, rt, rn, ARMREG_IP0);
629 static inline __attribute__((warn_unused_result)) guint8*
630 emit_ldrswx (guint8 *code, int rt, int rn, int imm)
632 if (arm_is_pimm12_scaled (imm, 4)) {
633 arm_ldrswx (code, rt, rn, imm);
635 g_assert (rt != ARMREG_IP0);
636 g_assert (rn != ARMREG_IP0);
637 code = emit_imm (code, ARMREG_IP0, imm);
638 arm_ldrswx_reg (code, rt, rn, ARMREG_IP0);
643 static inline __attribute__((warn_unused_result)) guint8*
644 emit_ldrw (guint8 *code, int rt, int rn, int imm)
646 if (arm_is_pimm12_scaled (imm, 4)) {
647 arm_ldrw (code, rt, rn, imm);
649 g_assert (rn != ARMREG_IP0);
650 code = emit_imm (code, ARMREG_IP0, imm);
651 arm_ldrw_reg (code, rt, rn, ARMREG_IP0);
656 static inline __attribute__((warn_unused_result)) guint8*
657 emit_ldrx (guint8 *code, int rt, int rn, int imm)
659 if (arm_is_pimm12_scaled (imm, 8)) {
660 arm_ldrx (code, rt, rn, imm);
662 g_assert (rn != ARMREG_IP0);
663 code = emit_imm (code, ARMREG_IP0, imm);
664 arm_ldrx_reg (code, rt, rn, ARMREG_IP0);
669 static inline __attribute__((warn_unused_result)) guint8*
670 emit_ldrfpw (guint8 *code, int rt, int rn, int imm)
672 if (arm_is_pimm12_scaled (imm, 4)) {
673 arm_ldrfpw (code, rt, rn, imm);
675 g_assert (rn != ARMREG_IP0);
676 code = emit_imm (code, ARMREG_IP0, imm);
677 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
678 arm_ldrfpw (code, rt, ARMREG_IP0, 0);
683 static inline __attribute__((warn_unused_result)) guint8*
684 emit_ldrfpx (guint8 *code, int rt, int rn, int imm)
686 if (arm_is_pimm12_scaled (imm, 8)) {
687 arm_ldrfpx (code, rt, rn, imm);
689 g_assert (rn != ARMREG_IP0);
690 code = emit_imm (code, ARMREG_IP0, imm);
691 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
692 arm_ldrfpx (code, rt, ARMREG_IP0, 0);
698 mono_arm_emit_ldrx (guint8 *code, int rt, int rn, int imm)
700 return emit_ldrx (code, rt, rn, imm);
704 emit_call (MonoCompile *cfg, guint8* code, guint32 patch_type, gconstpointer data)
707 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_IMM);
708 code = emit_imm64_template (code, ARMREG_LR);
709 arm_blrx (code, ARMREG_LR);
711 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_BL);
713 cfg->thunk_area += THUNK_SIZE;
718 emit_aotconst_full (MonoCompile *cfg, MonoJumpInfo **ji, guint8 *code, guint8 *start, int dreg, guint32 patch_type, gconstpointer data)
721 mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
723 *ji = mono_patch_info_list_prepend (*ji, code - start, patch_type, data);
724 /* See arch_emit_got_access () in aot-compiler.c */
725 arm_ldrx_lit (code, dreg, 0);
732 emit_aotconst (MonoCompile *cfg, guint8 *code, int dreg, guint32 patch_type, gconstpointer data)
734 return emit_aotconst_full (cfg, NULL, code, NULL, dreg, patch_type, data);
738 * mono_arm_emit_aotconst:
740 * Emit code to load an AOT constant into DREG. Usable from trampolines.
743 mono_arm_emit_aotconst (gpointer ji, guint8 *code, guint8 *code_start, int dreg, guint32 patch_type, gconstpointer data)
745 return emit_aotconst_full (NULL, (MonoJumpInfo**)ji, code, code_start, dreg, patch_type, data);
749 emit_tls_get (guint8 *code, int dreg, int tls_offset)
751 arm_mrs (code, dreg, ARM_MRS_REG_TPIDR_EL0);
752 if (tls_offset < 256) {
753 arm_ldrx (code, dreg, dreg, tls_offset);
755 code = emit_addx_imm (code, dreg, dreg, tls_offset);
756 arm_ldrx (code, dreg, dreg, 0);
762 emit_tls_get_reg (guint8 *code, int dreg, int offset_reg)
764 g_assert (offset_reg != ARMREG_IP0);
765 arm_mrs (code, ARMREG_IP0, ARM_MRS_REG_TPIDR_EL0);
766 arm_ldrx_reg (code, dreg, ARMREG_IP0, offset_reg);
771 emit_tls_set (guint8 *code, int sreg, int tls_offset)
773 int tmpreg = ARMREG_IP0;
775 g_assert (sreg != tmpreg);
776 arm_mrs (code, tmpreg, ARM_MRS_REG_TPIDR_EL0);
777 if (tls_offset < 256) {
778 arm_strx (code, sreg, tmpreg, tls_offset);
780 code = emit_addx_imm (code, tmpreg, tmpreg, tls_offset);
781 arm_strx (code, sreg, tmpreg, 0);
788 emit_tls_set_reg (guint8 *code, int sreg, int offset_reg)
790 int tmpreg = ARMREG_IP0;
792 g_assert (sreg != tmpreg);
793 arm_mrs (code, tmpreg, ARM_MRS_REG_TPIDR_EL0);
794 arm_strx_reg (code, sreg, tmpreg, offset_reg);
801 * - ldrp [fp, lr], [sp], !stack_offfset
802 * Clobbers TEMP_REGS.
804 __attribute__((warn_unused_result)) guint8*
805 mono_arm_emit_destroy_frame (guint8 *code, int stack_offset, guint64 temp_regs)
807 arm_movspx (code, ARMREG_SP, ARMREG_FP);
809 if (arm_is_ldpx_imm (stack_offset)) {
810 arm_ldpx_post (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, stack_offset);
812 arm_ldpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
813 /* sp += stack_offset */
814 g_assert (temp_regs & (1 << ARMREG_IP0));
815 if (temp_regs & (1 << ARMREG_IP1)) {
816 code = emit_addx_sp_imm (code, stack_offset);
818 int imm = stack_offset;
820 /* Can't use addx_sp_imm () since we can't clobber ip0/ip1 */
821 arm_addx_imm (code, ARMREG_IP0, ARMREG_SP, 0);
823 arm_addx_imm (code, ARMREG_IP0, ARMREG_IP0, 256);
826 arm_addx_imm (code, ARMREG_SP, ARMREG_IP0, imm);
832 #define is_call_imm(diff) ((gint)(diff) >= -33554432 && (gint)(diff) <= 33554431)
835 emit_thunk (guint8 *code, gconstpointer target)
839 arm_ldrx_lit (code, ARMREG_IP0, code + 8);
840 arm_brx (code, ARMREG_IP0);
841 *(guint64*)code = (guint64)target;
842 code += sizeof (guint64);
844 mono_arch_flush_icache (p, code - p);
849 create_thunk (MonoCompile *cfg, MonoDomain *domain, guchar *code, const guchar *target)
852 MonoThunkJitInfo *info;
856 guint8 *target_thunk;
859 domain = mono_domain_get ();
863 * This can be called multiple times during JITting,
864 * save the current position in cfg->arch to avoid
865 * doing a O(n^2) search.
867 if (!cfg->arch.thunks) {
868 cfg->arch.thunks = cfg->thunks;
869 cfg->arch.thunks_size = cfg->thunk_area;
871 thunks = cfg->arch.thunks;
872 thunks_size = cfg->arch.thunks_size;
874 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, mono_method_full_name (cfg->method, TRUE));
875 g_assert_not_reached ();
878 g_assert (*(guint32*)thunks == 0);
879 emit_thunk (thunks, target);
881 cfg->arch.thunks += THUNK_SIZE;
882 cfg->arch.thunks_size -= THUNK_SIZE;
886 ji = mini_jit_info_table_find (domain, (char*)code, NULL);
888 info = mono_jit_info_get_thunk_info (ji);
891 thunks = (guint8*)ji->code_start + info->thunks_offset;
892 thunks_size = info->thunks_size;
894 orig_target = mono_arch_get_call_target (code + 4);
896 mono_domain_lock (domain);
899 if (orig_target >= thunks && orig_target < thunks + thunks_size) {
900 /* The call already points to a thunk, because of trampolines etc. */
901 target_thunk = orig_target;
903 for (p = thunks; p < thunks + thunks_size; p += THUNK_SIZE) {
904 if (((guint32*)p) [0] == 0) {
908 } else if (((guint64*)p) [1] == (guint64)target) {
909 /* Thunk already points to target */
916 //printf ("THUNK: %p %p %p\n", code, target, target_thunk);
919 mono_domain_unlock (domain);
920 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, cfg ? mono_method_full_name (cfg->method, TRUE) : mono_method_full_name (jinfo_get_method (ji), TRUE));
921 g_assert_not_reached ();
924 emit_thunk (target_thunk, target);
926 mono_domain_unlock (domain);
933 arm_patch_full (MonoCompile *cfg, MonoDomain *domain, guint8 *code, guint8 *target, int relocation)
935 switch (relocation) {
937 if (arm_is_bl_disp (code, target)) {
938 arm_b (code, target);
942 thunk = create_thunk (cfg, domain, code, target);
943 g_assert (arm_is_bl_disp (code, thunk));
947 case MONO_R_ARM64_BCC: {
950 cond = arm_get_bcc_cond (code);
951 arm_bcc (code, cond, target);
954 case MONO_R_ARM64_CBZ:
955 arm_set_cbz_target (code, target);
957 case MONO_R_ARM64_IMM: {
958 guint64 imm = (guint64)target;
961 /* emit_imm64_template () */
962 dreg = arm_get_movzx_rd (code);
963 arm_movzx (code, dreg, imm & 0xffff, 0);
964 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
965 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
966 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
969 case MONO_R_ARM64_BL:
970 if (arm_is_bl_disp (code, target)) {
971 arm_bl (code, target);
975 thunk = create_thunk (cfg, domain, code, target);
976 g_assert (arm_is_bl_disp (code, thunk));
977 arm_bl (code, thunk);
981 g_assert_not_reached ();
986 arm_patch_rel (guint8 *code, guint8 *target, int relocation)
988 arm_patch_full (NULL, NULL, code, target, relocation);
992 mono_arm_patch (guint8 *code, guint8 *target, int relocation)
994 arm_patch_rel (code, target, relocation);
998 mono_arch_patch_code_new (MonoCompile *cfg, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gpointer target)
1002 ip = ji->ip.i + code;
1005 case MONO_PATCH_INFO_METHOD_JUMP:
1006 /* ji->relocation is not set by the caller */
1007 arm_patch_rel (ip, (guint8*)target, MONO_R_ARM64_B);
1010 arm_patch_full (cfg, domain, ip, (guint8*)target, ji->relocation);
1016 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
1021 mono_arch_flush_register_windows (void)
1026 mono_arch_find_imt_method (mgreg_t *regs, guint8 *code)
1028 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1032 mono_arch_find_static_call_vtable (mgreg_t *regs, guint8 *code)
1034 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1038 mono_arch_context_get_int_reg (MonoContext *ctx, int reg)
1040 return ctx->regs [reg];
1044 mono_arch_context_set_int_reg (MonoContext *ctx, int reg, mgreg_t val)
1046 ctx->regs [reg] = val;
1050 * mono_arch_set_target:
1052 * Set the target architecture the JIT backend should generate code for, in the form
1053 * of a GNU target triplet. Only used in AOT mode.
1056 mono_arch_set_target (char *mtriple)
1058 if (strstr (mtriple, "darwin") || strstr (mtriple, "ios")) {
1064 add_general (CallInfo *cinfo, ArgInfo *ainfo, int size, gboolean sign)
1066 if (cinfo->gr >= PARAM_REGS) {
1067 ainfo->storage = ArgOnStack;
1069 /* Assume size == align */
1070 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1071 ainfo->offset = cinfo->stack_usage;
1072 ainfo->slot_size = size;
1074 cinfo->stack_usage += size;
1076 ainfo->offset = cinfo->stack_usage;
1077 ainfo->slot_size = 8;
1078 ainfo->sign = FALSE;
1079 /* Put arguments into 8 byte aligned stack slots */
1080 cinfo->stack_usage += 8;
1083 ainfo->storage = ArgInIReg;
1084 ainfo->reg = cinfo->gr;
1090 add_fp (CallInfo *cinfo, ArgInfo *ainfo, gboolean single)
1092 int size = single ? 4 : 8;
1094 if (cinfo->fr >= FP_PARAM_REGS) {
1095 ainfo->storage = single ? ArgOnStackR4 : ArgOnStackR8;
1097 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1098 ainfo->offset = cinfo->stack_usage;
1099 ainfo->slot_size = size;
1100 cinfo->stack_usage += size;
1102 ainfo->offset = cinfo->stack_usage;
1103 ainfo->slot_size = 8;
1104 /* Put arguments into 8 byte aligned stack slots */
1105 cinfo->stack_usage += 8;
1109 ainfo->storage = ArgInFRegR4;
1111 ainfo->storage = ArgInFReg;
1112 ainfo->reg = cinfo->fr;
1118 is_hfa (MonoType *t, int *out_nfields, int *out_esize, int *field_offsets)
1122 MonoClassField *field;
1123 MonoType *ftype, *prev_ftype = NULL;
1126 klass = mono_class_from_mono_type (t);
1128 while ((field = mono_class_get_fields (klass, &iter))) {
1129 if (field->type->attrs & FIELD_ATTRIBUTE_STATIC)
1131 ftype = mono_field_get_type (field);
1132 ftype = mini_get_underlying_type (ftype);
1134 if (MONO_TYPE_ISSTRUCT (ftype)) {
1135 int nested_nfields, nested_esize;
1136 int nested_field_offsets [16];
1138 if (!is_hfa (ftype, &nested_nfields, &nested_esize, nested_field_offsets))
1140 if (nested_esize == 4)
1141 ftype = &mono_defaults.single_class->byval_arg;
1143 ftype = &mono_defaults.double_class->byval_arg;
1144 if (prev_ftype && prev_ftype->type != ftype->type)
1147 for (i = 0; i < nested_nfields; ++i) {
1148 if (nfields + i < 4)
1149 field_offsets [nfields + i] = field->offset - sizeof (MonoObject) + nested_field_offsets [i];
1151 nfields += nested_nfields;
1153 if (!(!ftype->byref && (ftype->type == MONO_TYPE_R4 || ftype->type == MONO_TYPE_R8)))
1155 if (prev_ftype && prev_ftype->type != ftype->type)
1159 field_offsets [nfields] = field->offset - sizeof (MonoObject);
1163 if (nfields == 0 || nfields > 4)
1165 *out_nfields = nfields;
1166 *out_esize = prev_ftype->type == MONO_TYPE_R4 ? 4 : 8;
1171 add_valuetype (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1173 int i, size, align_size, nregs, nfields, esize;
1174 int field_offsets [16];
1177 size = mini_type_stack_size_full (t, &align, cinfo->pinvoke);
1178 align_size = ALIGN_TO (size, 8);
1180 nregs = align_size / 8;
1181 if (is_hfa (t, &nfields, &esize, field_offsets)) {
1183 * The struct might include nested float structs aligned at 8,
1184 * so need to keep track of the offsets of the individual fields.
1186 if (cinfo->fr + nfields <= FP_PARAM_REGS) {
1187 ainfo->storage = ArgHFA;
1188 ainfo->reg = cinfo->fr;
1189 ainfo->nregs = nfields;
1191 ainfo->esize = esize;
1192 for (i = 0; i < nfields; ++i)
1193 ainfo->foffsets [i] = field_offsets [i];
1194 cinfo->fr += ainfo->nregs;
1196 ainfo->nfregs_to_skip = FP_PARAM_REGS > cinfo->fr ? FP_PARAM_REGS - cinfo->fr : 0;
1197 cinfo->fr = FP_PARAM_REGS;
1198 size = ALIGN_TO (size, 8);
1199 ainfo->storage = ArgVtypeOnStack;
1200 ainfo->offset = cinfo->stack_usage;
1203 ainfo->nregs = nfields;
1204 ainfo->esize = esize;
1205 cinfo->stack_usage += size;
1210 if (align_size > 16) {
1211 ainfo->storage = ArgVtypeByRef;
1216 if (cinfo->gr + nregs > PARAM_REGS) {
1217 size = ALIGN_TO (size, 8);
1218 ainfo->storage = ArgVtypeOnStack;
1219 ainfo->offset = cinfo->stack_usage;
1221 cinfo->stack_usage += size;
1222 cinfo->gr = PARAM_REGS;
1224 ainfo->storage = ArgVtypeInIRegs;
1225 ainfo->reg = cinfo->gr;
1226 ainfo->nregs = nregs;
1233 add_param (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1237 ptype = mini_get_underlying_type (t);
1238 switch (ptype->type) {
1240 add_general (cinfo, ainfo, 1, TRUE);
1242 case MONO_TYPE_BOOLEAN:
1244 add_general (cinfo, ainfo, 1, FALSE);
1247 add_general (cinfo, ainfo, 2, TRUE);
1250 case MONO_TYPE_CHAR:
1251 add_general (cinfo, ainfo, 2, FALSE);
1254 add_general (cinfo, ainfo, 4, TRUE);
1257 add_general (cinfo, ainfo, 4, FALSE);
1262 case MONO_TYPE_FNPTR:
1263 case MONO_TYPE_CLASS:
1264 case MONO_TYPE_OBJECT:
1265 case MONO_TYPE_SZARRAY:
1266 case MONO_TYPE_ARRAY:
1267 case MONO_TYPE_STRING:
1270 add_general (cinfo, ainfo, 8, FALSE);
1273 add_fp (cinfo, ainfo, FALSE);
1276 add_fp (cinfo, ainfo, TRUE);
1278 case MONO_TYPE_VALUETYPE:
1279 case MONO_TYPE_TYPEDBYREF:
1280 add_valuetype (cinfo, ainfo, ptype);
1282 case MONO_TYPE_VOID:
1283 ainfo->storage = ArgNone;
1285 case MONO_TYPE_GENERICINST:
1286 if (!mono_type_generic_inst_is_valuetype (ptype)) {
1287 add_general (cinfo, ainfo, 8, FALSE);
1288 } else if (mini_is_gsharedvt_variable_type (ptype)) {
1290 * Treat gsharedvt arguments as large vtypes
1292 ainfo->storage = ArgVtypeByRef;
1293 ainfo->gsharedvt = TRUE;
1295 add_valuetype (cinfo, ainfo, ptype);
1299 case MONO_TYPE_MVAR:
1300 g_assert (mini_is_gsharedvt_type (ptype));
1301 ainfo->storage = ArgVtypeByRef;
1302 ainfo->gsharedvt = TRUE;
1305 g_assert_not_reached ();
1313 * Obtain information about a call according to the calling convention.
1316 get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
1320 int n, pstart, pindex;
1322 n = sig->hasthis + sig->param_count;
1325 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1327 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1330 cinfo->pinvoke = sig->pinvoke;
1333 add_param (cinfo, &cinfo->ret, sig->ret);
1334 if (cinfo->ret.storage == ArgVtypeByRef)
1335 cinfo->ret.reg = ARMREG_R8;
1339 cinfo->stack_usage = 0;
1343 add_general (cinfo, cinfo->args + 0, 8, FALSE);
1345 for (pindex = pstart; pindex < sig->param_count; ++pindex) {
1346 ainfo = cinfo->args + sig->hasthis + pindex;
1348 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1349 /* Prevent implicit arguments and sig_cookie from
1350 being passed in registers */
1351 cinfo->gr = PARAM_REGS;
1352 cinfo->fr = FP_PARAM_REGS;
1353 /* Emit the signature cookie just before the implicit arguments */
1354 add_param (cinfo, &cinfo->sig_cookie, &mono_defaults.int_class->byval_arg);
1357 add_param (cinfo, ainfo, sig->params [pindex]);
1358 if (ainfo->storage == ArgVtypeByRef) {
1359 /* Pass the argument address in the next register */
1360 if (cinfo->gr >= PARAM_REGS) {
1361 ainfo->storage = ArgVtypeByRefOnStack;
1362 ainfo->offset = cinfo->stack_usage;
1363 cinfo->stack_usage += 8;
1365 ainfo->reg = cinfo->gr;
1371 /* Handle the case where there are no implicit arguments */
1372 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1373 /* Prevent implicit arguments and sig_cookie from
1374 being passed in registers */
1375 cinfo->gr = PARAM_REGS;
1376 cinfo->fr = FP_PARAM_REGS;
1377 /* Emit the signature cookie just before the implicit arguments */
1378 add_param (cinfo, &cinfo->sig_cookie, &mono_defaults.int_class->byval_arg);
1381 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, MONO_ARCH_FRAME_ALIGNMENT);
1387 MonoMethodSignature *sig;
1390 MonoType **param_types;
1391 int n_fpargs, n_fpret;
1395 dyn_call_supported (CallInfo *cinfo, MonoMethodSignature *sig)
1399 if (sig->hasthis + sig->param_count > PARAM_REGS + DYN_CALL_STACK_ARGS)
1402 // FIXME: Add more cases
1403 switch (cinfo->ret.storage) {
1410 case ArgVtypeInIRegs:
1411 if (cinfo->ret.nregs > 2)
1420 for (i = 0; i < cinfo->nargs; ++i) {
1421 ArgInfo *ainfo = &cinfo->args [i];
1423 switch (ainfo->storage) {
1425 case ArgVtypeInIRegs:
1432 if (ainfo->offset >= DYN_CALL_STACK_ARGS * sizeof (mgreg_t))
1444 mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
1446 ArchDynCallInfo *info;
1450 cinfo = get_call_info (NULL, sig);
1452 if (!dyn_call_supported (cinfo, sig)) {
1457 info = g_new0 (ArchDynCallInfo, 1);
1458 // FIXME: Preprocess the info to speed up start_dyn_call ()
1460 info->cinfo = cinfo;
1461 info->rtype = mini_get_underlying_type (sig->ret);
1462 info->param_types = g_new0 (MonoType*, sig->param_count);
1463 for (i = 0; i < sig->param_count; ++i)
1464 info->param_types [i] = mini_get_underlying_type (sig->params [i]);
1466 switch (cinfo->ret.storage) {
1472 info->n_fpret = cinfo->ret.nregs;
1478 return (MonoDynCallInfo*)info;
1482 mono_arch_dyn_call_free (MonoDynCallInfo *info)
1484 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1486 g_free (ainfo->cinfo);
1487 g_free (ainfo->param_types);
1492 bitcast_r4_to_r8 (float f)
1500 bitcast_r8_to_r4 (double f)
1508 mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, guint8 *buf, int buf_len)
1510 ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
1511 DynCallArgs *p = (DynCallArgs*)buf;
1512 int aindex, arg_index, greg, i, pindex;
1513 MonoMethodSignature *sig = dinfo->sig;
1514 CallInfo *cinfo = dinfo->cinfo;
1515 int buffer_offset = 0;
1517 g_assert (buf_len >= sizeof (DynCallArgs));
1521 p->n_fpargs = dinfo->n_fpargs;
1522 p->n_fpret = dinfo->n_fpret;
1529 p->regs [greg ++] = (mgreg_t)*(args [arg_index ++]);
1531 if (cinfo->ret.storage == ArgVtypeByRef)
1532 p->regs [ARMREG_R8] = (mgreg_t)ret;
1534 for (aindex = pindex; aindex < sig->param_count; aindex++) {
1535 MonoType *t = dinfo->param_types [aindex];
1536 gpointer *arg = args [arg_index ++];
1537 ArgInfo *ainfo = &cinfo->args [aindex + sig->hasthis];
1540 if (ainfo->storage == ArgOnStack) {
1541 slot = PARAM_REGS + 1 + (ainfo->offset / sizeof (mgreg_t));
1547 p->regs [slot] = (mgreg_t)*arg;
1551 if (ios_abi && ainfo->storage == ArgOnStack) {
1552 guint8 *stack_arg = (guint8*)&(p->regs [PARAM_REGS + 1]) + ainfo->offset;
1553 gboolean handled = TRUE;
1555 /* Special case arguments smaller than 1 machine word */
1557 case MONO_TYPE_BOOLEAN:
1559 *(guint8*)stack_arg = *(guint8*)arg;
1562 *(gint8*)stack_arg = *(gint8*)arg;
1565 case MONO_TYPE_CHAR:
1566 *(guint16*)stack_arg = *(guint16*)arg;
1569 *(gint16*)stack_arg = *(gint16*)arg;
1572 *(gint32*)stack_arg = *(gint32*)arg;
1575 *(guint32*)stack_arg = *(guint32*)arg;
1586 case MONO_TYPE_STRING:
1587 case MONO_TYPE_CLASS:
1588 case MONO_TYPE_ARRAY:
1589 case MONO_TYPE_SZARRAY:
1590 case MONO_TYPE_OBJECT:
1596 p->regs [slot] = (mgreg_t)*arg;
1598 case MONO_TYPE_BOOLEAN:
1600 p->regs [slot] = *(guint8*)arg;
1603 p->regs [slot] = *(gint8*)arg;
1606 p->regs [slot] = *(gint16*)arg;
1609 case MONO_TYPE_CHAR:
1610 p->regs [slot] = *(guint16*)arg;
1613 p->regs [slot] = *(gint32*)arg;
1616 p->regs [slot] = *(guint32*)arg;
1619 p->fpregs [ainfo->reg] = bitcast_r4_to_r8 (*(float*)arg);
1623 p->fpregs [ainfo->reg] = *(double*)arg;
1626 case MONO_TYPE_GENERICINST:
1627 if (MONO_TYPE_IS_REFERENCE (t)) {
1628 p->regs [slot] = (mgreg_t)*arg;
1631 if (t->type == MONO_TYPE_GENERICINST && mono_class_is_nullable (mono_class_from_mono_type (t))) {
1632 MonoClass *klass = mono_class_from_mono_type (t);
1633 guint8 *nullable_buf;
1637 * Use p->buffer as a temporary buffer since the data needs to be available after this call
1638 * if the nullable param is passed by ref.
1640 size = mono_class_value_size (klass, NULL);
1641 nullable_buf = p->buffer + buffer_offset;
1642 buffer_offset += size;
1643 g_assert (buffer_offset <= 256);
1645 /* The argument pointed to by arg is either a boxed vtype or null */
1646 mono_nullable_init (nullable_buf, (MonoObject*)arg, klass);
1648 arg = (gpointer*)nullable_buf;
1654 case MONO_TYPE_VALUETYPE:
1655 switch (ainfo->storage) {
1656 case ArgVtypeInIRegs:
1657 for (i = 0; i < ainfo->nregs; ++i)
1658 p->regs [slot ++] = ((mgreg_t*)arg) [i];
1661 if (ainfo->esize == 4) {
1662 for (i = 0; i < ainfo->nregs; ++i)
1663 p->fpregs [ainfo->reg + i] = bitcast_r4_to_r8 (((float*)arg) [ainfo->foffsets [i] / 4]);
1665 for (i = 0; i < ainfo->nregs; ++i)
1666 p->fpregs [ainfo->reg + i] = ((double*)arg) [ainfo->foffsets [i] / 8];
1668 p->n_fpargs += ainfo->nregs;
1671 p->regs [slot] = (mgreg_t)arg;
1674 g_assert_not_reached ();
1679 g_assert_not_reached ();
1685 mono_arch_finish_dyn_call (MonoDynCallInfo *info, guint8 *buf)
1687 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1688 CallInfo *cinfo = ainfo->cinfo;
1689 DynCallArgs *args = (DynCallArgs*)buf;
1690 MonoType *ptype = ainfo->rtype;
1691 guint8 *ret = args->ret;
1692 mgreg_t res = args->res;
1693 mgreg_t res2 = args->res2;
1696 if (cinfo->ret.storage == ArgVtypeByRef)
1699 switch (ptype->type) {
1700 case MONO_TYPE_VOID:
1701 *(gpointer*)ret = NULL;
1703 case MONO_TYPE_STRING:
1704 case MONO_TYPE_CLASS:
1705 case MONO_TYPE_ARRAY:
1706 case MONO_TYPE_SZARRAY:
1707 case MONO_TYPE_OBJECT:
1711 *(gpointer*)ret = (gpointer)res;
1717 case MONO_TYPE_BOOLEAN:
1718 *(guint8*)ret = res;
1721 *(gint16*)ret = res;
1724 case MONO_TYPE_CHAR:
1725 *(guint16*)ret = res;
1728 *(gint32*)ret = res;
1731 *(guint32*)ret = res;
1735 *(guint64*)ret = res;
1738 *(float*)ret = bitcast_r8_to_r4 (args->fpregs [0]);
1741 *(double*)ret = args->fpregs [0];
1743 case MONO_TYPE_GENERICINST:
1744 if (MONO_TYPE_IS_REFERENCE (ptype)) {
1745 *(gpointer*)ret = (gpointer)res;
1750 case MONO_TYPE_VALUETYPE:
1751 switch (ainfo->cinfo->ret.storage) {
1752 case ArgVtypeInIRegs:
1753 *(mgreg_t*)ret = res;
1754 if (ainfo->cinfo->ret.nregs > 1)
1755 ((mgreg_t*)ret) [1] = res2;
1758 /* Use the same area for returning fp values */
1759 if (cinfo->ret.esize == 4) {
1760 for (i = 0; i < cinfo->ret.nregs; ++i)
1761 ((float*)ret) [cinfo->ret.foffsets [i] / 4] = bitcast_r8_to_r4 (args->fpregs [i]);
1763 for (i = 0; i < cinfo->ret.nregs; ++i)
1764 ((double*)ret) [cinfo->ret.foffsets [i] / 8] = args->fpregs [i];
1768 g_assert_not_reached ();
1773 g_assert_not_reached ();
1778 void sys_icache_invalidate (void *start, size_t len);
1782 mono_arch_flush_icache (guint8 *code, gint size)
1784 #ifndef MONO_CROSS_COMPILE
1786 sys_icache_invalidate (code, size);
1788 /* Don't rely on GCC's __clear_cache implementation, as it caches
1789 * icache/dcache cache line sizes, that can vary between cores on
1790 * big.LITTLE architectures. */
1791 guint64 end = (guint64) (code + size);
1792 guint64 addr, ctr_el0;
1793 static size_t icache_line_size = 0xffff, dcache_line_size = 0xffff;
1794 size_t isize, dsize;
1796 asm volatile ("mrs %0, ctr_el0" : "=r" (ctr_el0));
1797 isize = 4 << ((ctr_el0 >> 0 ) & 0xf);
1798 dsize = 4 << ((ctr_el0 >> 16) & 0xf);
1800 /* determine the global minimum cache line size */
1801 icache_line_size = isize = MIN (icache_line_size, isize);
1802 dcache_line_size = dsize = MIN (dcache_line_size, dsize);
1804 addr = (guint64) code & ~(guint64) (dsize - 1);
1805 for (; addr < end; addr += dsize)
1806 asm volatile("dc civac, %0" : : "r" (addr) : "memory");
1807 asm volatile("dsb ish" : : : "memory");
1809 addr = (guint64) code & ~(guint64) (isize - 1);
1810 for (; addr < end; addr += isize)
1811 asm volatile("ic ivau, %0" : : "r" (addr) : "memory");
1813 asm volatile ("dsb ish" : : : "memory");
1814 asm volatile ("isb" : : : "memory");
1822 mono_arch_opcode_needs_emulation (MonoCompile *cfg, int opcode)
1829 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
1834 for (i = 0; i < cfg->num_varinfo; i++) {
1835 MonoInst *ins = cfg->varinfo [i];
1836 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
1839 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
1842 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
1843 (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
1846 if (mono_is_regsize_var (ins->inst_vtype)) {
1847 g_assert (MONO_VARINFO (cfg, i)->reg == -1);
1848 g_assert (i == vmv->idx);
1849 vars = g_list_prepend (vars, vmv);
1853 vars = mono_varlist_sort (cfg, vars, 0);
1859 mono_arch_get_global_int_regs (MonoCompile *cfg)
1864 /* r28 is reserved for cfg->arch.args_reg */
1865 /* r27 is reserved for the imt argument */
1866 for (i = ARMREG_R19; i <= ARMREG_R26; ++i)
1867 regs = g_list_prepend (regs, GUINT_TO_POINTER (i));
1873 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
1875 MonoInst *ins = cfg->varinfo [vmv->idx];
1877 if (ins->opcode == OP_ARG)
1884 mono_arch_create_vars (MonoCompile *cfg)
1886 MonoMethodSignature *sig;
1889 sig = mono_method_signature (cfg->method);
1890 if (!cfg->arch.cinfo)
1891 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
1892 cinfo = cfg->arch.cinfo;
1894 if (cinfo->ret.storage == ArgVtypeByRef) {
1895 cfg->vret_addr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1896 cfg->vret_addr->flags |= MONO_INST_VOLATILE;
1899 if (cfg->gen_sdb_seq_points) {
1902 if (cfg->compile_aot) {
1903 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1904 ins->flags |= MONO_INST_VOLATILE;
1905 cfg->arch.seq_point_info_var = ins;
1908 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1909 ins->flags |= MONO_INST_VOLATILE;
1910 cfg->arch.ss_tramp_var = ins;
1912 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1913 ins->flags |= MONO_INST_VOLATILE;
1914 cfg->arch.bp_tramp_var = ins;
1917 if (cfg->method->save_lmf) {
1918 cfg->create_lmf_var = TRUE;
1921 cfg->lmf_ir_mono_lmf = TRUE;
1927 mono_arch_allocate_vars (MonoCompile *cfg)
1929 MonoMethodSignature *sig;
1933 int i, offset, size, align;
1934 guint32 locals_stack_size, locals_stack_align;
1938 * Allocate arguments and locals to either register (OP_REGVAR) or to a stack slot (OP_REGOFFSET).
1939 * Compute cfg->stack_offset and update cfg->used_int_regs.
1942 sig = mono_method_signature (cfg->method);
1944 if (!cfg->arch.cinfo)
1945 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
1946 cinfo = cfg->arch.cinfo;
1949 * The ARM64 ABI always uses a frame pointer.
1950 * The instruction set prefers positive offsets, so fp points to the bottom of the
1951 * frame, and stack slots are at positive offsets.
1952 * If some arguments are received on the stack, their offsets relative to fp can
1953 * not be computed right now because the stack frame might grow due to spilling
1954 * done by the local register allocator. To solve this, we reserve a register
1955 * which points to them.
1956 * The stack frame looks like this:
1957 * args_reg -> <bottom of parent frame>
1959 * fp -> <saved fp+lr>
1960 * sp -> <localloc/params area>
1962 cfg->frame_reg = ARMREG_FP;
1963 cfg->flags |= MONO_CFG_HAS_SPILLUP;
1969 if (cinfo->stack_usage) {
1970 g_assert (!(cfg->used_int_regs & (1 << ARMREG_R28)));
1971 cfg->arch.args_reg = ARMREG_R28;
1972 cfg->used_int_regs |= 1 << ARMREG_R28;
1975 if (cfg->method->save_lmf) {
1976 /* The LMF var is allocated normally */
1978 /* Callee saved regs */
1979 cfg->arch.saved_gregs_offset = offset;
1980 for (i = 0; i < 32; ++i)
1981 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) && (cfg->used_int_regs & (1 << i)))
1986 switch (cinfo->ret.storage) {
1992 cfg->ret->opcode = OP_REGVAR;
1993 cfg->ret->dreg = cinfo->ret.reg;
1995 case ArgVtypeInIRegs:
1997 /* Allocate a local to hold the result, the epilog will copy it to the correct place */
1998 cfg->ret->opcode = OP_REGOFFSET;
1999 cfg->ret->inst_basereg = cfg->frame_reg;
2000 cfg->ret->inst_offset = offset;
2001 if (cinfo->ret.storage == ArgHFA)
2008 /* This variable will be initalized in the prolog from R8 */
2009 cfg->vret_addr->opcode = OP_REGOFFSET;
2010 cfg->vret_addr->inst_basereg = cfg->frame_reg;
2011 cfg->vret_addr->inst_offset = offset;
2013 if (G_UNLIKELY (cfg->verbose_level > 1)) {
2014 printf ("vret_addr =");
2015 mono_print_ins (cfg->vret_addr);
2019 g_assert_not_reached ();
2024 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2025 ainfo = cinfo->args + i;
2027 ins = cfg->args [i];
2028 if (ins->opcode == OP_REGVAR)
2031 ins->opcode = OP_REGOFFSET;
2032 ins->inst_basereg = cfg->frame_reg;
2034 switch (ainfo->storage) {
2038 // FIXME: Use nregs/size
2039 /* These will be copied to the stack in the prolog */
2040 ins->inst_offset = offset;
2046 case ArgVtypeOnStack:
2047 /* These are in the parent frame */
2048 g_assert (cfg->arch.args_reg);
2049 ins->inst_basereg = cfg->arch.args_reg;
2050 ins->inst_offset = ainfo->offset;
2052 case ArgVtypeInIRegs:
2054 ins->opcode = OP_REGOFFSET;
2055 ins->inst_basereg = cfg->frame_reg;
2056 /* These arguments are saved to the stack in the prolog */
2057 ins->inst_offset = offset;
2058 if (cfg->verbose_level >= 2)
2059 printf ("arg %d allocated to %s+0x%0x.\n", i, mono_arch_regname (ins->inst_basereg), (int)ins->inst_offset);
2060 if (ainfo->storage == ArgHFA)
2066 case ArgVtypeByRefOnStack: {
2069 if (ainfo->gsharedvt) {
2070 ins->opcode = OP_REGOFFSET;
2071 ins->inst_basereg = cfg->arch.args_reg;
2072 ins->inst_offset = ainfo->offset;
2076 /* The vtype address is in the parent frame */
2077 g_assert (cfg->arch.args_reg);
2078 MONO_INST_NEW (cfg, vtaddr, 0);
2079 vtaddr->opcode = OP_REGOFFSET;
2080 vtaddr->inst_basereg = cfg->arch.args_reg;
2081 vtaddr->inst_offset = ainfo->offset;
2083 /* Need an indirection */
2084 ins->opcode = OP_VTARG_ADDR;
2085 ins->inst_left = vtaddr;
2088 case ArgVtypeByRef: {
2091 if (ainfo->gsharedvt) {
2092 ins->opcode = OP_REGOFFSET;
2093 ins->inst_basereg = cfg->frame_reg;
2094 ins->inst_offset = offset;
2099 /* The vtype address is in a register, will be copied to the stack in the prolog */
2100 MONO_INST_NEW (cfg, vtaddr, 0);
2101 vtaddr->opcode = OP_REGOFFSET;
2102 vtaddr->inst_basereg = cfg->frame_reg;
2103 vtaddr->inst_offset = offset;
2106 /* Need an indirection */
2107 ins->opcode = OP_VTARG_ADDR;
2108 ins->inst_left = vtaddr;
2112 g_assert_not_reached ();
2117 /* Allocate these first so they have a small offset, OP_SEQ_POINT depends on this */
2118 // FIXME: Allocate these to registers
2119 ins = cfg->arch.seq_point_info_var;
2123 offset += align - 1;
2124 offset &= ~(align - 1);
2125 ins->opcode = OP_REGOFFSET;
2126 ins->inst_basereg = cfg->frame_reg;
2127 ins->inst_offset = offset;
2130 ins = cfg->arch.ss_tramp_var;
2134 offset += align - 1;
2135 offset &= ~(align - 1);
2136 ins->opcode = OP_REGOFFSET;
2137 ins->inst_basereg = cfg->frame_reg;
2138 ins->inst_offset = offset;
2141 ins = cfg->arch.bp_tramp_var;
2145 offset += align - 1;
2146 offset &= ~(align - 1);
2147 ins->opcode = OP_REGOFFSET;
2148 ins->inst_basereg = cfg->frame_reg;
2149 ins->inst_offset = offset;
2154 offsets = mono_allocate_stack_slots (cfg, FALSE, &locals_stack_size, &locals_stack_align);
2155 if (locals_stack_align)
2156 offset = ALIGN_TO (offset, locals_stack_align);
2158 for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
2159 if (offsets [i] != -1) {
2160 ins = cfg->varinfo [i];
2161 ins->opcode = OP_REGOFFSET;
2162 ins->inst_basereg = cfg->frame_reg;
2163 ins->inst_offset = offset + offsets [i];
2164 //printf ("allocated local %d to ", i); mono_print_tree_nl (ins);
2167 offset += locals_stack_size;
2169 offset = ALIGN_TO (offset, MONO_ARCH_FRAME_ALIGNMENT);
2171 cfg->stack_offset = offset;
2176 mono_arch_get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
2181 LLVMCallInfo *linfo;
2183 n = sig->param_count + sig->hasthis;
2185 cinfo = get_call_info (cfg->mempool, sig);
2187 linfo = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
2189 switch (cinfo->ret.storage) {
2196 linfo->ret.storage = LLVMArgVtypeByRef;
2199 // FIXME: This doesn't work yet since the llvm backend represents these types as an i8
2200 // array which is returned in int regs
2203 linfo->ret.storage = LLVMArgFpStruct;
2204 linfo->ret.nslots = cinfo->ret.nregs;
2205 linfo->ret.esize = cinfo->ret.esize;
2207 case ArgVtypeInIRegs:
2208 /* LLVM models this by returning an int */
2209 linfo->ret.storage = LLVMArgVtypeAsScalar;
2210 linfo->ret.nslots = cinfo->ret.nregs;
2211 linfo->ret.esize = cinfo->ret.esize;
2214 g_assert_not_reached ();
2218 for (i = 0; i < n; ++i) {
2219 LLVMArgInfo *lainfo = &linfo->args [i];
2221 ainfo = cinfo->args + i;
2223 lainfo->storage = LLVMArgNone;
2225 switch (ainfo->storage) {
2232 lainfo->storage = LLVMArgNormal;
2235 case ArgVtypeByRefOnStack:
2236 lainfo->storage = LLVMArgVtypeByRef;
2241 lainfo->storage = LLVMArgAsFpArgs;
2242 lainfo->nslots = ainfo->nregs;
2243 lainfo->esize = ainfo->esize;
2244 for (j = 0; j < ainfo->nregs; ++j)
2245 lainfo->pair_storage [j] = LLVMArgInFPReg;
2248 case ArgVtypeInIRegs:
2249 lainfo->storage = LLVMArgAsIArgs;
2250 lainfo->nslots = ainfo->nregs;
2252 case ArgVtypeOnStack:
2256 lainfo->storage = LLVMArgAsFpArgs;
2257 lainfo->nslots = ainfo->nregs;
2258 lainfo->esize = ainfo->esize;
2259 lainfo->ndummy_fpargs = ainfo->nfregs_to_skip;
2260 for (j = 0; j < ainfo->nregs; ++j)
2261 lainfo->pair_storage [j] = LLVMArgInFPReg;
2263 lainfo->storage = LLVMArgAsIArgs;
2264 lainfo->nslots = ainfo->size / 8;
2268 g_assert_not_reached ();
2278 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, ArgStorage storage, int reg, MonoInst *arg)
2284 MONO_INST_NEW (cfg, ins, OP_MOVE);
2285 ins->dreg = mono_alloc_ireg_copy (cfg, arg->dreg);
2286 ins->sreg1 = arg->dreg;
2287 MONO_ADD_INS (cfg->cbb, ins);
2288 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, FALSE);
2291 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2292 ins->dreg = mono_alloc_freg (cfg);
2293 ins->sreg1 = arg->dreg;
2294 MONO_ADD_INS (cfg->cbb, ins);
2295 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2298 if (COMPILE_LLVM (cfg))
2299 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2301 MONO_INST_NEW (cfg, ins, OP_RMOVE);
2303 MONO_INST_NEW (cfg, ins, OP_ARM_SETFREG_R4);
2304 ins->dreg = mono_alloc_freg (cfg);
2305 ins->sreg1 = arg->dreg;
2306 MONO_ADD_INS (cfg->cbb, ins);
2307 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2310 g_assert_not_reached ();
2316 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
2318 MonoMethodSignature *tmp_sig;
2321 if (call->tail_call)
2324 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
2327 * mono_ArgIterator_Setup assumes the signature cookie is
2328 * passed first and all the arguments which were before it are
2329 * passed on the stack after the signature. So compensate by
2330 * passing a different signature.
2332 tmp_sig = mono_metadata_signature_dup (call->signature);
2333 tmp_sig->param_count -= call->signature->sentinelpos;
2334 tmp_sig->sentinelpos = 0;
2335 memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
2337 sig_reg = mono_alloc_ireg (cfg);
2338 MONO_EMIT_NEW_SIGNATURECONST (cfg, sig_reg, tmp_sig);
2340 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, cinfo->sig_cookie.offset, sig_reg);
2344 mono_arch_emit_call (MonoCompile *cfg, MonoCallInst *call)
2346 MonoMethodSignature *sig;
2347 MonoInst *arg, *vtarg;
2352 sig = call->signature;
2354 cinfo = get_call_info (cfg->mempool, sig);
2356 switch (cinfo->ret.storage) {
2357 case ArgVtypeInIRegs:
2360 * The vtype is returned in registers, save the return area address in a local, and save the vtype into
2361 * the location pointed to by it after call in emit_move_return_value ().
2363 if (!cfg->arch.vret_addr_loc) {
2364 cfg->arch.vret_addr_loc = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2365 /* Prevent it from being register allocated or optimized away */
2366 ((MonoInst*)cfg->arch.vret_addr_loc)->flags |= MONO_INST_VOLATILE;
2369 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, ((MonoInst*)cfg->arch.vret_addr_loc)->dreg, call->vret_var->dreg);
2372 /* Pass the vtype return address in R8 */
2373 MONO_INST_NEW (cfg, vtarg, OP_MOVE);
2374 vtarg->sreg1 = call->vret_var->dreg;
2375 vtarg->dreg = mono_alloc_preg (cfg);
2376 MONO_ADD_INS (cfg->cbb, vtarg);
2378 mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
2384 for (i = 0; i < cinfo->nargs; ++i) {
2385 ainfo = cinfo->args + i;
2386 arg = call->args [i];
2388 if ((sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
2389 /* Emit the signature cookie just before the implicit arguments */
2390 emit_sig_cookie (cfg, call, cinfo);
2393 switch (ainfo->storage) {
2397 add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, arg);
2400 switch (ainfo->slot_size) {
2402 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2405 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2408 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2411 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2414 g_assert_not_reached ();
2419 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER8_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2422 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2424 case ArgVtypeInIRegs:
2426 case ArgVtypeByRefOnStack:
2427 case ArgVtypeOnStack:
2433 size = mono_class_value_size (arg->klass, &align);
2435 MONO_INST_NEW (cfg, ins, OP_OUTARG_VT);
2436 ins->sreg1 = arg->dreg;
2437 ins->klass = arg->klass;
2438 ins->backend.size = size;
2439 ins->inst_p0 = call;
2440 ins->inst_p1 = mono_mempool_alloc (cfg->mempool, sizeof (ArgInfo));
2441 memcpy (ins->inst_p1, ainfo, sizeof (ArgInfo));
2442 MONO_ADD_INS (cfg->cbb, ins);
2446 g_assert_not_reached ();
2451 /* Handle the case where there are no implicit arguments */
2452 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (cinfo->nargs == sig->sentinelpos))
2453 emit_sig_cookie (cfg, call, cinfo);
2455 call->call_info = cinfo;
2456 call->stack_usage = cinfo->stack_usage;
2460 mono_arch_emit_outarg_vt (MonoCompile *cfg, MonoInst *ins, MonoInst *src)
2462 MonoCallInst *call = (MonoCallInst*)ins->inst_p0;
2463 ArgInfo *ainfo = ins->inst_p1;
2467 if (ins->backend.size == 0 && !ainfo->gsharedvt)
2470 switch (ainfo->storage) {
2471 case ArgVtypeInIRegs:
2472 for (i = 0; i < ainfo->nregs; ++i) {
2473 // FIXME: Smaller sizes
2474 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2475 load->dreg = mono_alloc_ireg (cfg);
2476 load->inst_basereg = src->dreg;
2477 load->inst_offset = i * sizeof(mgreg_t);
2478 MONO_ADD_INS (cfg->cbb, load);
2479 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg + i, load);
2483 for (i = 0; i < ainfo->nregs; ++i) {
2484 if (ainfo->esize == 4)
2485 MONO_INST_NEW (cfg, load, OP_LOADR4_MEMBASE);
2487 MONO_INST_NEW (cfg, load, OP_LOADR8_MEMBASE);
2488 load->dreg = mono_alloc_freg (cfg);
2489 load->inst_basereg = src->dreg;
2490 load->inst_offset = ainfo->foffsets [i];
2491 MONO_ADD_INS (cfg->cbb, load);
2492 add_outarg_reg (cfg, call, ainfo->esize == 4 ? ArgInFRegR4 : ArgInFReg, ainfo->reg + i, load);
2496 case ArgVtypeByRefOnStack: {
2497 MonoInst *vtaddr, *load, *arg;
2499 /* Pass the vtype address in a reg/on the stack */
2500 if (ainfo->gsharedvt) {
2503 /* Make a copy of the argument */
2504 vtaddr = mono_compile_create_var (cfg, &ins->klass->byval_arg, OP_LOCAL);
2506 MONO_INST_NEW (cfg, load, OP_LDADDR);
2507 load->inst_p0 = vtaddr;
2508 vtaddr->flags |= MONO_INST_INDIRECT;
2509 load->type = STACK_MP;
2510 load->klass = vtaddr->klass;
2511 load->dreg = mono_alloc_ireg (cfg);
2512 MONO_ADD_INS (cfg->cbb, load);
2513 mini_emit_memcpy (cfg, load->dreg, 0, src->dreg, 0, ainfo->size, 8);
2516 if (ainfo->storage == ArgVtypeByRef) {
2517 MONO_INST_NEW (cfg, arg, OP_MOVE);
2518 arg->dreg = mono_alloc_preg (cfg);
2519 arg->sreg1 = load->dreg;
2520 MONO_ADD_INS (cfg->cbb, arg);
2521 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg, arg);
2523 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, load->dreg);
2527 case ArgVtypeOnStack:
2528 for (i = 0; i < ainfo->size / 8; ++i) {
2529 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2530 load->dreg = mono_alloc_ireg (cfg);
2531 load->inst_basereg = src->dreg;
2532 load->inst_offset = i * 8;
2533 MONO_ADD_INS (cfg->cbb, load);
2534 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI8_MEMBASE_REG, ARMREG_SP, ainfo->offset + (i * 8), load->dreg);
2538 g_assert_not_reached ();
2544 mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
2546 MonoMethodSignature *sig;
2549 sig = mono_method_signature (cfg->method);
2550 if (!cfg->arch.cinfo)
2551 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
2552 cinfo = cfg->arch.cinfo;
2554 switch (cinfo->ret.storage) {
2558 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, cfg->ret->dreg, val->dreg);
2561 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2564 if (COMPILE_LLVM (cfg))
2565 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2567 MONO_EMIT_NEW_UNALU (cfg, OP_RMOVE, cfg->ret->dreg, val->dreg);
2569 MONO_EMIT_NEW_UNALU (cfg, OP_ARM_SETFREG_R4, cfg->ret->dreg, val->dreg);
2572 g_assert_not_reached ();
2578 mono_arch_tail_call_supported (MonoCompile *cfg, MonoMethodSignature *caller_sig, MonoMethodSignature *callee_sig)
2583 if (cfg->compile_aot && !cfg->full_aot)
2584 /* OP_TAILCALL doesn't work with AOT */
2587 c1 = get_call_info (NULL, caller_sig);
2588 c2 = get_call_info (NULL, callee_sig);
2590 // FIXME: Relax these restrictions
2591 if (c1->stack_usage != 0)
2593 if (c1->stack_usage != c2->stack_usage)
2595 if ((c1->ret.storage != ArgNone && c1->ret.storage != ArgInIReg) || c1->ret.storage != c2->ret.storage)
2605 mono_arch_is_inst_imm (gint64 imm)
2607 return (imm >= -((gint64)1<<31) && imm <= (((gint64)1<<31)-1));
2611 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
2618 mono_arch_instrument_epilog_full (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments, gboolean preserve_argument_registers)
2625 mono_arch_peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
2631 mono_arch_peephole_pass_2 (MonoCompile *cfg, MonoBasicBlock *bb)
2636 #define ADD_NEW_INS(cfg,dest,op) do { \
2637 MONO_INST_NEW ((cfg), (dest), (op)); \
2638 mono_bblock_insert_before_ins (bb, ins, (dest)); \
2642 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
2644 MonoInst *ins, *temp, *last_ins = NULL;
2646 MONO_BB_FOR_EACH_INS (bb, ins) {
2647 switch (ins->opcode) {
2652 if (ins->next && (ins->next->opcode == OP_COND_EXC_C || ins->next->opcode == OP_COND_EXC_IC))
2653 /* ARM sets the C flag to 1 if there was _no_ overflow */
2654 ins->next->opcode = OP_COND_EXC_NC;
2658 case OP_IDIV_UN_IMM:
2659 case OP_IREM_UN_IMM:
2661 mono_decompose_op_imm (cfg, bb, ins);
2663 case OP_LOCALLOC_IMM:
2664 if (ins->inst_imm > 32) {
2665 ADD_NEW_INS (cfg, temp, OP_ICONST);
2666 temp->inst_c0 = ins->inst_imm;
2667 temp->dreg = mono_alloc_ireg (cfg);
2668 ins->sreg1 = temp->dreg;
2669 ins->opcode = mono_op_imm_to_op (ins->opcode);
2672 case OP_ICOMPARE_IMM:
2673 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBEQ) {
2674 ins->next->opcode = OP_ARM64_CBZW;
2675 ins->next->sreg1 = ins->sreg1;
2677 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBNE_UN) {
2678 ins->next->opcode = OP_ARM64_CBNZW;
2679 ins->next->sreg1 = ins->sreg1;
2683 case OP_LCOMPARE_IMM:
2684 case OP_COMPARE_IMM:
2685 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBEQ) {
2686 ins->next->opcode = OP_ARM64_CBZX;
2687 ins->next->sreg1 = ins->sreg1;
2689 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBNE_UN) {
2690 ins->next->opcode = OP_ARM64_CBNZX;
2691 ins->next->sreg1 = ins->sreg1;
2696 gboolean swap = FALSE;
2700 /* Optimized away */
2706 * FP compares with unordered operands set the flags
2707 * to NZCV=0011, which matches some non-unordered compares
2708 * as well, like LE, so have to swap the operands.
2710 switch (ins->next->opcode) {
2712 ins->next->opcode = OP_FBGT;
2716 ins->next->opcode = OP_FBGE;
2724 ins->sreg1 = ins->sreg2;
2735 bb->last_ins = last_ins;
2736 bb->max_vreg = cfg->next_vreg;
2740 mono_arch_decompose_long_opts (MonoCompile *cfg, MonoInst *long_ins)
2745 opcode_to_armcond (int opcode)
2756 case OP_COND_EXC_IEQ:
2757 case OP_COND_EXC_EQ:
2774 case OP_COND_EXC_IGT:
2775 case OP_COND_EXC_GT:
2790 case OP_COND_EXC_ILT:
2791 case OP_COND_EXC_LT:
2799 case OP_COND_EXC_INE_UN:
2800 case OP_COND_EXC_NE_UN:
2806 case OP_COND_EXC_IGE_UN:
2807 case OP_COND_EXC_GE_UN:
2817 case OP_COND_EXC_IGT_UN:
2818 case OP_COND_EXC_GT_UN:
2824 case OP_COND_EXC_ILE_UN:
2825 case OP_COND_EXC_LE_UN:
2833 case OP_COND_EXC_ILT_UN:
2834 case OP_COND_EXC_LT_UN:
2837 * FCMP sets the NZCV condition bits as follows:
2842 * ARMCOND_LT is N!=V, so it matches unordered too, so
2843 * fclt and fclt_un need to be special cased.
2853 case OP_COND_EXC_IC:
2855 case OP_COND_EXC_OV:
2856 case OP_COND_EXC_IOV:
2858 case OP_COND_EXC_NC:
2859 case OP_COND_EXC_INC:
2861 case OP_COND_EXC_NO:
2862 case OP_COND_EXC_INO:
2865 printf ("%s\n", mono_inst_name (opcode));
2866 g_assert_not_reached ();
2871 /* This clobbers LR */
2872 static inline __attribute__((warn_unused_result)) guint8*
2873 emit_cond_exc (MonoCompile *cfg, guint8 *code, int opcode, const char *exc_name)
2877 cond = opcode_to_armcond (opcode);
2879 arm_adrx (code, ARMREG_IP1, code);
2880 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_EXC, exc_name, MONO_R_ARM64_BCC);
2881 arm_bcc (code, cond, 0);
2886 emit_move_return_value (MonoCompile *cfg, guint8 * code, MonoInst *ins)
2891 call = (MonoCallInst*)ins;
2892 cinfo = call->call_info;
2894 switch (cinfo->ret.storage) {
2898 /* LLVM compiled code might only set the bottom bits */
2899 if (call->signature && mini_get_underlying_type (call->signature->ret)->type == MONO_TYPE_I4)
2900 arm_sxtwx (code, call->inst.dreg, cinfo->ret.reg);
2901 else if (call->inst.dreg != cinfo->ret.reg)
2902 arm_movx (code, call->inst.dreg, cinfo->ret.reg);
2905 if (call->inst.dreg != cinfo->ret.reg)
2906 arm_fmovd (code, call->inst.dreg, cinfo->ret.reg);
2910 arm_fmovs (code, call->inst.dreg, cinfo->ret.reg);
2912 arm_fcvt_sd (code, call->inst.dreg, cinfo->ret.reg);
2914 case ArgVtypeInIRegs: {
2915 MonoInst *loc = cfg->arch.vret_addr_loc;
2918 /* Load the destination address */
2919 g_assert (loc && loc->opcode == OP_REGOFFSET);
2920 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
2921 for (i = 0; i < cinfo->ret.nregs; ++i)
2922 arm_strx (code, cinfo->ret.reg + i, ARMREG_LR, i * 8);
2926 MonoInst *loc = cfg->arch.vret_addr_loc;
2929 /* Load the destination address */
2930 g_assert (loc && loc->opcode == OP_REGOFFSET);
2931 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
2932 for (i = 0; i < cinfo->ret.nregs; ++i) {
2933 if (cinfo->ret.esize == 4)
2934 arm_strfpw (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
2936 arm_strfpx (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
2943 g_assert_not_reached ();
2950 * emit_branch_island:
2952 * Emit a branch island for the conditional branches from cfg->native_code + start_offset to code.
2955 emit_branch_island (MonoCompile *cfg, guint8 *code, int start_offset)
2958 int offset, island_size;
2960 /* Iterate over the patch infos added so far by this bb */
2962 for (ji = cfg->patch_info; ji; ji = ji->next) {
2963 if (ji->ip.i < start_offset)
2964 /* The patch infos are in reverse order, so this means the end */
2966 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ)
2971 offset = code - cfg->native_code;
2972 if (offset > (cfg->code_size - island_size - 16)) {
2973 cfg->code_size *= 2;
2974 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
2975 code = cfg->native_code + offset;
2978 /* Branch over the island */
2979 arm_b (code, code + 4 + island_size);
2981 for (ji = cfg->patch_info; ji; ji = ji->next) {
2982 if (ji->ip.i < start_offset)
2984 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ) {
2985 /* Rewrite the cond branch so it branches to an uncoditional branch in the branch island */
2986 arm_patch_rel (cfg->native_code + ji->ip.i, code, ji->relocation);
2987 /* Rewrite the patch so it points to the unconditional branch */
2988 ji->ip.i = code - cfg->native_code;
2989 ji->relocation = MONO_R_ARM64_B;
2998 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
3003 guint8 *code = cfg->native_code + cfg->code_len;
3004 int start_offset, max_len, dreg, sreg1, sreg2;
3007 if (cfg->verbose_level > 2)
3008 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
3010 start_offset = code - cfg->native_code;
3012 MONO_BB_FOR_EACH_INS (bb, ins) {
3013 offset = code - cfg->native_code;
3015 max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
3017 if (offset > (cfg->code_size - max_len - 16)) {
3018 cfg->code_size *= 2;
3019 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
3020 code = cfg->native_code + offset;
3023 if (G_UNLIKELY (cfg->arch.cond_branch_islands && offset - start_offset > 4 * 0x1ffff)) {
3024 /* Emit a branch island for large basic blocks */
3025 code = emit_branch_island (cfg, code, start_offset);
3026 offset = code - cfg->native_code;
3027 start_offset = offset;
3030 mono_debug_record_line_number (cfg, ins, offset);
3035 imm = ins->inst_imm;
3037 switch (ins->opcode) {
3039 code = emit_imm (code, dreg, ins->inst_c0);
3042 code = emit_imm64 (code, dreg, ins->inst_c0);
3046 arm_movx (code, dreg, sreg1);
3049 case OP_RELAXED_NOP:
3052 mono_add_patch_info_rel (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0, MONO_R_ARM64_IMM);
3053 code = emit_imm64_template (code, dreg);
3057 * gdb does not like encountering the hw breakpoint ins in the debugged code.
3058 * So instead of emitting a trap, we emit a call a C function and place a
3061 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, (gpointer)"mono_break");
3066 arm_addx_imm (code, ARMREG_IP0, sreg1, (MONO_ARCH_FRAME_ALIGNMENT - 1));
3067 // FIXME: andx_imm doesn't work yet
3068 code = emit_imm (code, ARMREG_IP1, -MONO_ARCH_FRAME_ALIGNMENT);
3069 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3070 //arm_andx_imm (code, ARMREG_IP0, sreg1, - MONO_ARCH_FRAME_ALIGNMENT);
3071 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
3072 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
3073 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
3076 /* ip1 = pointer, ip0 = end */
3077 arm_addx (code, ARMREG_IP0, ARMREG_IP1, ARMREG_IP0);
3079 arm_cmpx (code, ARMREG_IP1, ARMREG_IP0);
3081 arm_bcc (code, ARMCOND_EQ, 0);
3082 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_IP1, 0);
3083 arm_addx_imm (code, ARMREG_IP1, ARMREG_IP1, 16);
3084 arm_b (code, buf [0]);
3085 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3087 arm_movspx (code, dreg, ARMREG_SP);
3088 if (cfg->param_area)
3089 code = emit_subx_sp_imm (code, cfg->param_area);
3092 case OP_LOCALLOC_IMM: {
3095 imm = ALIGN_TO (ins->inst_imm, MONO_ARCH_FRAME_ALIGNMENT);
3096 g_assert (arm_is_arith_imm (imm));
3097 arm_subx_imm (code, ARMREG_SP, ARMREG_SP, imm);
3100 g_assert (MONO_ARCH_FRAME_ALIGNMENT == 16);
3102 while (offset < imm) {
3103 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_SP, offset);
3106 arm_movspx (code, dreg, ARMREG_SP);
3107 if (cfg->param_area)
3108 code = emit_subx_sp_imm (code, cfg->param_area);
3112 code = emit_aotconst (cfg, code, dreg, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3114 case OP_OBJC_GET_SELECTOR:
3115 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_OBJC_SELECTOR_REF, ins->inst_p0);
3116 /* See arch_emit_objc_selector_ref () in aot-compiler.c */
3117 arm_ldrx_lit (code, ins->dreg, 0);
3121 case OP_SEQ_POINT: {
3122 MonoInst *info_var = cfg->arch.seq_point_info_var;
3125 * For AOT, we use one got slot per method, which will point to a
3126 * SeqPointInfo structure, containing all the information required
3127 * by the code below.
3129 if (cfg->compile_aot) {
3130 g_assert (info_var);
3131 g_assert (info_var->opcode == OP_REGOFFSET);
3134 if (ins->flags & MONO_INST_SINGLE_STEP_LOC) {
3135 MonoInst *var = cfg->arch.ss_tramp_var;
3138 g_assert (var->opcode == OP_REGOFFSET);
3139 /* Load ss_tramp_var */
3140 /* This is equal to &ss_trampoline */
3141 arm_ldrx (code, ARMREG_IP1, var->inst_basereg, var->inst_offset);
3142 /* Load the trampoline address */
3143 arm_ldrx (code, ARMREG_IP1, ARMREG_IP1, 0);
3144 /* Call it if it is non-null */
3145 arm_cbzx (code, ARMREG_IP1, code + 8);
3146 arm_blrx (code, ARMREG_IP1);
3149 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
3151 if (cfg->compile_aot) {
3152 guint32 offset = code - cfg->native_code;
3155 arm_ldrx (code, ARMREG_IP1, info_var->inst_basereg, info_var->inst_offset);
3156 /* Add the offset */
3157 val = ((offset / 4) * sizeof (guint8*)) + MONO_STRUCT_OFFSET (SeqPointInfo, bp_addrs);
3158 /* Load the info->bp_addrs [offset], which is either 0 or the address of the bp trampoline */
3159 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP1, val);
3160 /* Skip the load if its 0 */
3161 arm_cbzx (code, ARMREG_IP1, code + 8);
3162 /* Call the breakpoint trampoline */
3163 arm_blrx (code, ARMREG_IP1);
3165 MonoInst *var = cfg->arch.bp_tramp_var;
3168 g_assert (var->opcode == OP_REGOFFSET);
3169 /* Load the address of the bp trampoline into IP0 */
3170 arm_ldrx (code, ARMREG_IP0, var->inst_basereg, var->inst_offset);
3172 * A placeholder for a possible breakpoint inserted by
3173 * mono_arch_set_breakpoint ().
3182 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_B);
3186 arm_brx (code, sreg1);
3218 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3219 cond = opcode_to_armcond (ins->opcode);
3220 arm_bcc (code, cond, 0);
3224 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3225 /* For fp compares, ARMCOND_LT is lt or unordered */
3226 arm_bcc (code, ARMCOND_LT, 0);
3229 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3230 arm_bcc (code, ARMCOND_EQ, 0);
3231 offset = code - cfg->native_code;
3232 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3233 /* For fp compares, ARMCOND_LT is lt or unordered */
3234 arm_bcc (code, ARMCOND_LT, 0);
3237 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3238 arm_cbzw (code, sreg1, 0);
3241 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3242 arm_cbzx (code, sreg1, 0);
3244 case OP_ARM64_CBNZW:
3245 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3246 arm_cbnzw (code, sreg1, 0);
3248 case OP_ARM64_CBNZX:
3249 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3250 arm_cbnzx (code, sreg1, 0);
3254 arm_addw (code, dreg, sreg1, sreg2);
3257 arm_addx (code, dreg, sreg1, sreg2);
3260 arm_subw (code, dreg, sreg1, sreg2);
3263 arm_subx (code, dreg, sreg1, sreg2);
3266 arm_andw (code, dreg, sreg1, sreg2);
3269 arm_andx (code, dreg, sreg1, sreg2);
3272 arm_orrw (code, dreg, sreg1, sreg2);
3275 arm_orrx (code, dreg, sreg1, sreg2);
3278 arm_eorw (code, dreg, sreg1, sreg2);
3281 arm_eorx (code, dreg, sreg1, sreg2);
3284 arm_negw (code, dreg, sreg1);
3287 arm_negx (code, dreg, sreg1);
3290 arm_mvnw (code, dreg, sreg1);
3293 arm_mvnx (code, dreg, sreg1);
3296 arm_addsw (code, dreg, sreg1, sreg2);
3300 arm_addsx (code, dreg, sreg1, sreg2);
3303 arm_subsw (code, dreg, sreg1, sreg2);
3307 arm_subsx (code, dreg, sreg1, sreg2);
3310 arm_cmpw (code, sreg1, sreg2);
3314 arm_cmpx (code, sreg1, sreg2);
3317 code = emit_addw_imm (code, dreg, sreg1, imm);
3321 code = emit_addx_imm (code, dreg, sreg1, imm);
3324 code = emit_subw_imm (code, dreg, sreg1, imm);
3327 code = emit_subx_imm (code, dreg, sreg1, imm);
3330 code = emit_andw_imm (code, dreg, sreg1, imm);
3334 code = emit_andx_imm (code, dreg, sreg1, imm);
3337 code = emit_orrw_imm (code, dreg, sreg1, imm);
3340 code = emit_orrx_imm (code, dreg, sreg1, imm);
3343 code = emit_eorw_imm (code, dreg, sreg1, imm);
3346 code = emit_eorx_imm (code, dreg, sreg1, imm);
3348 case OP_ICOMPARE_IMM:
3349 code = emit_cmpw_imm (code, sreg1, imm);
3351 case OP_LCOMPARE_IMM:
3352 case OP_COMPARE_IMM:
3354 arm_cmpx (code, sreg1, ARMREG_RZR);
3356 // FIXME: 32 vs 64 bit issues for 0xffffffff
3357 code = emit_imm64 (code, ARMREG_LR, imm);
3358 arm_cmpx (code, sreg1, ARMREG_LR);
3362 arm_lslvw (code, dreg, sreg1, sreg2);
3365 arm_lslvx (code, dreg, sreg1, sreg2);
3368 arm_asrvw (code, dreg, sreg1, sreg2);
3371 arm_asrvx (code, dreg, sreg1, sreg2);
3374 arm_lsrvw (code, dreg, sreg1, sreg2);
3377 arm_lsrvx (code, dreg, sreg1, sreg2);
3381 arm_movx (code, dreg, sreg1);
3383 arm_lslw (code, dreg, sreg1, imm);
3387 arm_movx (code, dreg, sreg1);
3389 arm_lslx (code, dreg, sreg1, imm);
3393 arm_movx (code, dreg, sreg1);
3395 arm_asrw (code, dreg, sreg1, imm);
3400 arm_movx (code, dreg, sreg1);
3402 arm_asrx (code, dreg, sreg1, imm);
3404 case OP_ISHR_UN_IMM:
3406 arm_movx (code, dreg, sreg1);
3408 arm_lsrw (code, dreg, sreg1, imm);
3411 case OP_LSHR_UN_IMM:
3413 arm_movx (code, dreg, sreg1);
3415 arm_lsrx (code, dreg, sreg1, imm);
3420 arm_sxtwx (code, dreg, sreg1);
3423 /* Clean out the upper word */
3424 arm_movw (code, dreg, sreg1);
3427 arm_lslx (code, dreg, sreg1, imm);
3430 /* MULTIPLY/DIVISION */
3433 // FIXME: Optimize this
3434 /* Check for zero */
3435 arm_cmpx_imm (code, sreg2, 0);
3436 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3437 /* Check for INT_MIN/-1 */
3438 code = emit_imm (code, ARMREG_IP0, 0x80000000);
3439 arm_cmpx (code, sreg1, ARMREG_IP0);
3440 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3441 code = emit_imm (code, ARMREG_IP0, 0xffffffff);
3442 arm_cmpx (code, sreg2, ARMREG_IP0);
3443 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3444 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3445 arm_cmpx_imm (code, ARMREG_IP0, 1);
3446 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "OverflowException");
3447 if (ins->opcode == OP_IREM) {
3448 arm_sdivw (code, ARMREG_LR, sreg1, sreg2);
3449 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3451 arm_sdivw (code, dreg, sreg1, sreg2);
3455 arm_cmpx_imm (code, sreg2, 0);
3456 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3457 arm_udivw (code, dreg, sreg1, sreg2);
3460 arm_cmpx_imm (code, sreg2, 0);
3461 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3462 arm_udivw (code, ARMREG_LR, sreg1, sreg2);
3463 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3467 // FIXME: Optimize this
3468 /* Check for zero */
3469 arm_cmpx_imm (code, sreg2, 0);
3470 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3471 /* Check for INT64_MIN/-1 */
3472 code = emit_imm64 (code, ARMREG_IP0, 0x8000000000000000);
3473 arm_cmpx (code, sreg1, ARMREG_IP0);
3474 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3475 code = emit_imm64 (code, ARMREG_IP0, 0xffffffffffffffff);
3476 arm_cmpx (code, sreg2, ARMREG_IP0);
3477 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3478 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3479 arm_cmpx_imm (code, ARMREG_IP0, 1);
3480 /* 64 bit uses ArithmeticException */
3481 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "ArithmeticException");
3482 if (ins->opcode == OP_LREM) {
3483 arm_sdivx (code, ARMREG_LR, sreg1, sreg2);
3484 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3486 arm_sdivx (code, dreg, sreg1, sreg2);
3490 arm_cmpx_imm (code, sreg2, 0);
3491 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3492 arm_udivx (code, dreg, sreg1, sreg2);
3495 arm_cmpx_imm (code, sreg2, 0);
3496 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3497 arm_udivx (code, ARMREG_LR, sreg1, sreg2);
3498 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3501 arm_mulw (code, dreg, sreg1, sreg2);
3504 arm_mulx (code, dreg, sreg1, sreg2);
3507 code = emit_imm (code, ARMREG_LR, imm);
3508 arm_mulw (code, dreg, sreg1, ARMREG_LR);
3512 code = emit_imm (code, ARMREG_LR, imm);
3513 arm_mulx (code, dreg, sreg1, ARMREG_LR);
3517 case OP_ICONV_TO_I1:
3518 case OP_LCONV_TO_I1:
3519 arm_sxtbx (code, dreg, sreg1);
3521 case OP_ICONV_TO_I2:
3522 case OP_LCONV_TO_I2:
3523 arm_sxthx (code, dreg, sreg1);
3525 case OP_ICONV_TO_U1:
3526 case OP_LCONV_TO_U1:
3527 arm_uxtbw (code, dreg, sreg1);
3529 case OP_ICONV_TO_U2:
3530 case OP_LCONV_TO_U2:
3531 arm_uxthw (code, dreg, sreg1);
3557 cond = opcode_to_armcond (ins->opcode);
3558 arm_cset (code, cond, dreg);
3571 cond = opcode_to_armcond (ins->opcode);
3572 arm_fcmpd (code, sreg1, sreg2);
3573 arm_cset (code, cond, dreg);
3578 case OP_LOADI1_MEMBASE:
3579 code = emit_ldrsbx (code, dreg, ins->inst_basereg, ins->inst_offset);
3581 case OP_LOADU1_MEMBASE:
3582 code = emit_ldrb (code, dreg, ins->inst_basereg, ins->inst_offset);
3584 case OP_LOADI2_MEMBASE:
3585 code = emit_ldrshx (code, dreg, ins->inst_basereg, ins->inst_offset);
3587 case OP_LOADU2_MEMBASE:
3588 code = emit_ldrh (code, dreg, ins->inst_basereg, ins->inst_offset);
3590 case OP_LOADI4_MEMBASE:
3591 code = emit_ldrswx (code, dreg, ins->inst_basereg, ins->inst_offset);
3593 case OP_LOADU4_MEMBASE:
3594 code = emit_ldrw (code, dreg, ins->inst_basereg, ins->inst_offset);
3596 case OP_LOAD_MEMBASE:
3597 case OP_LOADI8_MEMBASE:
3598 code = emit_ldrx (code, dreg, ins->inst_basereg, ins->inst_offset);
3600 case OP_STOREI1_MEMBASE_IMM:
3601 case OP_STOREI2_MEMBASE_IMM:
3602 case OP_STOREI4_MEMBASE_IMM:
3603 case OP_STORE_MEMBASE_IMM:
3604 case OP_STOREI8_MEMBASE_IMM: {
3608 code = emit_imm (code, ARMREG_LR, imm);
3611 immreg = ARMREG_RZR;
3614 switch (ins->opcode) {
3615 case OP_STOREI1_MEMBASE_IMM:
3616 code = emit_strb (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3618 case OP_STOREI2_MEMBASE_IMM:
3619 code = emit_strh (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3621 case OP_STOREI4_MEMBASE_IMM:
3622 code = emit_strw (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3624 case OP_STORE_MEMBASE_IMM:
3625 case OP_STOREI8_MEMBASE_IMM:
3626 code = emit_strx (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3629 g_assert_not_reached ();
3634 case OP_STOREI1_MEMBASE_REG:
3635 code = emit_strb (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3637 case OP_STOREI2_MEMBASE_REG:
3638 code = emit_strh (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3640 case OP_STOREI4_MEMBASE_REG:
3641 code = emit_strw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3643 case OP_STORE_MEMBASE_REG:
3644 case OP_STOREI8_MEMBASE_REG:
3645 code = emit_strx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3649 code = emit_tls_get (code, dreg, ins->inst_offset);
3651 case OP_TLS_GET_REG:
3652 code = emit_tls_get_reg (code, dreg, sreg1);
3655 code = emit_tls_set (code, sreg1, ins->inst_offset);
3657 case OP_TLS_SET_REG:
3658 code = emit_tls_set_reg (code, sreg1, sreg2);
3662 case OP_MEMORY_BARRIER:
3665 case OP_ATOMIC_ADD_I4: {
3669 arm_ldxrw (code, ARMREG_IP0, sreg1);
3670 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3671 arm_stlxrw (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3672 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3675 arm_movx (code, dreg, ARMREG_IP0);
3678 case OP_ATOMIC_ADD_I8: {
3682 arm_ldxrx (code, ARMREG_IP0, sreg1);
3683 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3684 arm_stlxrx (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3685 arm_cbnzx (code, ARMREG_IP1, buf [0]);
3688 arm_movx (code, dreg, ARMREG_IP0);
3691 case OP_ATOMIC_EXCHANGE_I4: {
3695 arm_ldxrw (code, ARMREG_IP0, sreg1);
3696 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3697 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3700 arm_movx (code, dreg, ARMREG_IP0);
3703 case OP_ATOMIC_EXCHANGE_I8: {
3707 arm_ldxrx (code, ARMREG_IP0, sreg1);
3708 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3709 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3712 arm_movx (code, dreg, ARMREG_IP0);
3715 case OP_ATOMIC_CAS_I4: {
3718 /* sreg2 is the value, sreg3 is the comparand */
3720 arm_ldxrw (code, ARMREG_IP0, sreg1);
3721 arm_cmpw (code, ARMREG_IP0, ins->sreg3);
3723 arm_bcc (code, ARMCOND_NE, 0);
3724 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3725 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3726 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3729 arm_movx (code, dreg, ARMREG_IP0);
3732 case OP_ATOMIC_CAS_I8: {
3736 arm_ldxrx (code, ARMREG_IP0, sreg1);
3737 arm_cmpx (code, ARMREG_IP0, ins->sreg3);
3739 arm_bcc (code, ARMCOND_NE, 0);
3740 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3741 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3742 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3745 arm_movx (code, dreg, ARMREG_IP0);
3748 case OP_ATOMIC_LOAD_I1: {
3749 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3750 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3752 arm_ldarb (code, ins->dreg, ARMREG_LR);
3753 arm_sxtbx (code, ins->dreg, ins->dreg);
3756 case OP_ATOMIC_LOAD_U1: {
3757 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3758 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3760 arm_ldarb (code, ins->dreg, ARMREG_LR);
3761 arm_uxtbx (code, ins->dreg, ins->dreg);
3764 case OP_ATOMIC_LOAD_I2: {
3765 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3766 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3768 arm_ldarh (code, ins->dreg, ARMREG_LR);
3769 arm_sxthx (code, ins->dreg, ins->dreg);
3772 case OP_ATOMIC_LOAD_U2: {
3773 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3774 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3776 arm_ldarh (code, ins->dreg, ARMREG_LR);
3777 arm_uxthx (code, ins->dreg, ins->dreg);
3780 case OP_ATOMIC_LOAD_I4: {
3781 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3782 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3784 arm_ldarw (code, ins->dreg, ARMREG_LR);
3785 arm_sxtwx (code, ins->dreg, ins->dreg);
3788 case OP_ATOMIC_LOAD_U4: {
3789 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3790 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3792 arm_ldarw (code, ins->dreg, ARMREG_LR);
3793 arm_movw (code, ins->dreg, ins->dreg); /* Clear upper half of the register. */
3796 case OP_ATOMIC_LOAD_I8:
3797 case OP_ATOMIC_LOAD_U8: {
3798 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3799 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3801 arm_ldarx (code, ins->dreg, ARMREG_LR);
3804 case OP_ATOMIC_LOAD_R4: {
3805 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3806 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3809 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3810 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3812 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3813 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3814 arm_fcvt_sd (code, ins->dreg, FP_TEMP_REG);
3818 case OP_ATOMIC_LOAD_R8: {
3819 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3820 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3822 arm_ldarx (code, ARMREG_LR, ARMREG_LR);
3823 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3826 case OP_ATOMIC_STORE_I1:
3827 case OP_ATOMIC_STORE_U1: {
3828 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3829 arm_stlrb (code, ARMREG_LR, ins->sreg1);
3830 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3834 case OP_ATOMIC_STORE_I2:
3835 case OP_ATOMIC_STORE_U2: {
3836 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3837 arm_stlrh (code, ARMREG_LR, ins->sreg1);
3838 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3842 case OP_ATOMIC_STORE_I4:
3843 case OP_ATOMIC_STORE_U4: {
3844 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3845 arm_stlrw (code, ARMREG_LR, ins->sreg1);
3846 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3850 case OP_ATOMIC_STORE_I8:
3851 case OP_ATOMIC_STORE_U8: {
3852 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3853 arm_stlrx (code, ARMREG_LR, ins->sreg1);
3854 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3858 case OP_ATOMIC_STORE_R4: {
3859 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3861 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
3862 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
3864 arm_fcvt_ds (code, FP_TEMP_REG, ins->sreg1);
3865 arm_fmov_double_to_rx (code, ARMREG_IP0, FP_TEMP_REG);
3866 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
3868 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3872 case OP_ATOMIC_STORE_R8: {
3873 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3874 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
3875 arm_stlrx (code, ARMREG_LR, ARMREG_IP0);
3876 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3883 guint64 imm = *(guint64*)ins->inst_p0;
3886 arm_fmov_rx_to_double (code, dreg, ARMREG_RZR);
3888 code = emit_imm64 (code, ARMREG_LR, imm);
3889 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3894 guint64 imm = *(guint32*)ins->inst_p0;
3896 code = emit_imm64 (code, ARMREG_LR, imm);
3898 arm_fmov_rx_to_double (code, dreg, ARMREG_LR);
3900 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3901 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3905 case OP_LOADR8_MEMBASE:
3906 code = emit_ldrfpx (code, dreg, ins->inst_basereg, ins->inst_offset);
3908 case OP_LOADR4_MEMBASE:
3910 code = emit_ldrfpw (code, dreg, ins->inst_basereg, ins->inst_offset);
3912 code = emit_ldrfpw (code, FP_TEMP_REG, ins->inst_basereg, ins->inst_offset);
3913 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3916 case OP_STORER8_MEMBASE_REG:
3917 code = emit_strfpx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3919 case OP_STORER4_MEMBASE_REG:
3921 code = emit_strfpw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3923 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
3924 code = emit_strfpw (code, FP_TEMP_REG, ins->inst_destbasereg, ins->inst_offset);
3929 arm_fmovd (code, dreg, sreg1);
3933 arm_fmovs (code, dreg, sreg1);
3935 case OP_MOVE_F_TO_I4:
3937 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
3939 arm_fcvt_ds (code, ins->dreg, ins->sreg1);
3940 arm_fmov_double_to_rx (code, ins->dreg, ins->dreg);
3943 case OP_MOVE_I4_TO_F:
3945 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3947 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3948 arm_fcvt_sd (code, ins->dreg, ins->dreg);
3951 case OP_MOVE_F_TO_I8:
3952 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
3954 case OP_MOVE_I8_TO_F:
3955 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3958 arm_fcmpd (code, sreg1, sreg2);
3961 arm_fcmps (code, sreg1, sreg2);
3963 case OP_FCONV_TO_I1:
3964 arm_fcvtzs_dx (code, dreg, sreg1);
3965 arm_sxtbx (code, dreg, dreg);
3967 case OP_FCONV_TO_U1:
3968 arm_fcvtzu_dx (code, dreg, sreg1);
3969 arm_uxtbw (code, dreg, dreg);
3971 case OP_FCONV_TO_I2:
3972 arm_fcvtzs_dx (code, dreg, sreg1);
3973 arm_sxthx (code, dreg, dreg);
3975 case OP_FCONV_TO_U2:
3976 arm_fcvtzu_dx (code, dreg, sreg1);
3977 arm_uxthw (code, dreg, dreg);
3979 case OP_FCONV_TO_I4:
3980 arm_fcvtzs_dx (code, dreg, sreg1);
3981 arm_sxtwx (code, dreg, dreg);
3983 case OP_FCONV_TO_U4:
3984 arm_fcvtzu_dx (code, dreg, sreg1);
3986 case OP_FCONV_TO_I8:
3987 arm_fcvtzs_dx (code, dreg, sreg1);
3989 case OP_FCONV_TO_U8:
3990 arm_fcvtzu_dx (code, dreg, sreg1);
3992 case OP_FCONV_TO_R4:
3994 arm_fcvt_ds (code, dreg, sreg1);
3996 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
3997 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4000 case OP_ICONV_TO_R4:
4002 arm_scvtf_rw_to_s (code, dreg, sreg1);
4004 arm_scvtf_rw_to_s (code, FP_TEMP_REG, sreg1);
4005 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4008 case OP_LCONV_TO_R4:
4010 arm_scvtf_rx_to_s (code, dreg, sreg1);
4012 arm_scvtf_rx_to_s (code, FP_TEMP_REG, sreg1);
4013 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
4016 case OP_ICONV_TO_R8:
4017 arm_scvtf_rw_to_d (code, dreg, sreg1);
4019 case OP_LCONV_TO_R8:
4020 arm_scvtf_rx_to_d (code, dreg, sreg1);
4022 case OP_ICONV_TO_R_UN:
4023 arm_ucvtf_rw_to_d (code, dreg, sreg1);
4025 case OP_LCONV_TO_R_UN:
4026 arm_ucvtf_rx_to_d (code, dreg, sreg1);
4029 arm_fadd_d (code, dreg, sreg1, sreg2);
4032 arm_fsub_d (code, dreg, sreg1, sreg2);
4035 arm_fmul_d (code, dreg, sreg1, sreg2);
4038 arm_fdiv_d (code, dreg, sreg1, sreg2);
4042 g_assert_not_reached ();
4045 arm_fneg_d (code, dreg, sreg1);
4047 case OP_ARM_SETFREG_R4:
4048 arm_fcvt_ds (code, dreg, sreg1);
4051 /* Check for infinity */
4052 code = emit_imm64 (code, ARMREG_LR, 0x7fefffffffffffffLL);
4053 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
4054 arm_fabs_d (code, FP_TEMP_REG2, sreg1);
4055 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG);
4056 code = emit_cond_exc (cfg, code, OP_COND_EXC_GT, "ArithmeticException");
4057 /* Check for nans */
4058 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG2);
4059 code = emit_cond_exc (cfg, code, OP_COND_EXC_OV, "ArithmeticException");
4060 arm_fmovd (code, dreg, sreg1);
4065 arm_fadd_s (code, dreg, sreg1, sreg2);
4068 arm_fsub_s (code, dreg, sreg1, sreg2);
4071 arm_fmul_s (code, dreg, sreg1, sreg2);
4074 arm_fdiv_s (code, dreg, sreg1, sreg2);
4077 arm_fneg_s (code, dreg, sreg1);
4079 case OP_RCONV_TO_I1:
4080 arm_fcvtzs_sx (code, dreg, sreg1);
4081 arm_sxtbx (code, dreg, dreg);
4083 case OP_RCONV_TO_U1:
4084 arm_fcvtzu_sx (code, dreg, sreg1);
4085 arm_uxtbw (code, dreg, dreg);
4087 case OP_RCONV_TO_I2:
4088 arm_fcvtzs_sx (code, dreg, sreg1);
4089 arm_sxthx (code, dreg, dreg);
4091 case OP_RCONV_TO_U2:
4092 arm_fcvtzu_sx (code, dreg, sreg1);
4093 arm_uxthw (code, dreg, dreg);
4095 case OP_RCONV_TO_I4:
4096 arm_fcvtzs_sx (code, dreg, sreg1);
4097 arm_sxtwx (code, dreg, dreg);
4099 case OP_RCONV_TO_U4:
4100 arm_fcvtzu_sx (code, dreg, sreg1);
4102 case OP_RCONV_TO_I8:
4103 arm_fcvtzs_sx (code, dreg, sreg1);
4105 case OP_RCONV_TO_U8:
4106 arm_fcvtzu_sx (code, dreg, sreg1);
4108 case OP_RCONV_TO_R8:
4109 arm_fcvt_sd (code, dreg, sreg1);
4111 case OP_RCONV_TO_R4:
4113 arm_fmovs (code, dreg, sreg1);
4125 cond = opcode_to_armcond (ins->opcode);
4126 arm_fcmps (code, sreg1, sreg2);
4127 arm_cset (code, cond, dreg);
4138 call = (MonoCallInst*)ins;
4139 if (ins->flags & MONO_INST_HAS_METHOD)
4140 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method);
4142 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr);
4143 code = emit_move_return_value (cfg, code, ins);
4145 case OP_VOIDCALL_REG:
4151 arm_blrx (code, sreg1);
4152 code = emit_move_return_value (cfg, code, ins);
4154 case OP_VOIDCALL_MEMBASE:
4155 case OP_CALL_MEMBASE:
4156 case OP_LCALL_MEMBASE:
4157 case OP_FCALL_MEMBASE:
4158 case OP_RCALL_MEMBASE:
4159 case OP_VCALL2_MEMBASE:
4160 code = emit_ldrx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4161 arm_blrx (code, ARMREG_IP0);
4162 code = emit_move_return_value (cfg, code, ins);
4165 MonoCallInst *call = (MonoCallInst*)ins;
4167 g_assert (!cfg->method->save_lmf);
4169 // FIXME: Copy stack arguments
4171 /* Restore registers */
4172 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4175 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, ((1 << ARMREG_IP0) | (1 << ARMREG_IP1)));
4177 if (cfg->compile_aot) {
4178 /* This is not a PLT patch */
4179 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_METHOD_JUMP, call->method);
4180 arm_brx (code, ARMREG_IP0);
4182 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, call->method, MONO_R_ARM64_B);
4185 ins->flags |= MONO_INST_GC_CALLSITE;
4186 ins->backend.pc_offset = code - cfg->native_code;
4190 g_assert (cfg->arch.cinfo);
4191 code = emit_addx_imm (code, ARMREG_IP0, cfg->arch.args_reg, ((CallInfo*)cfg->arch.cinfo)->sig_cookie.offset);
4192 arm_strx (code, ARMREG_IP0, sreg1, 0);
4195 MonoInst *var = cfg->dyn_call_var;
4196 guint8 *labels [16];
4200 * sreg1 points to a DynCallArgs structure initialized by mono_arch_start_dyn_call ().
4201 * sreg2 is the function to call.
4204 g_assert (var->opcode == OP_REGOFFSET);
4206 arm_movx (code, ARMREG_LR, sreg1);
4207 arm_movx (code, ARMREG_IP1, sreg2);
4209 /* Save args buffer */
4210 code = emit_strx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4212 /* Set fp argument regs */
4213 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpargs));
4214 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4216 arm_bcc (code, ARMCOND_EQ, 0);
4217 for (i = 0; i < 8; ++i)
4218 code = emit_ldrfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4219 arm_patch_rel (labels [0], code, MONO_R_ARM64_BCC);
4221 /* Set stack args */
4222 for (i = 0; i < DYN_CALL_STACK_ARGS; ++i) {
4223 code = emit_ldrx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, regs) + ((PARAM_REGS + 1 + i) * sizeof (mgreg_t)));
4224 code = emit_strx (code, ARMREG_R0, ARMREG_SP, i * sizeof (mgreg_t));
4227 /* Set argument registers + r8 */
4228 code = mono_arm_emit_load_regarray (code, 0x1ff, ARMREG_LR, 0);
4231 arm_blrx (code, ARMREG_IP1);
4234 code = emit_ldrx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4235 arm_strx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res));
4236 arm_strx (code, ARMREG_R1, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res2));
4237 /* Save fp result */
4238 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpret));
4239 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4241 arm_bcc (code, ARMCOND_EQ, 0);
4242 for (i = 0; i < 8; ++i)
4243 code = emit_strfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4244 arm_patch_rel (labels [1], code, MONO_R_ARM64_BCC);
4248 case OP_GENERIC_CLASS_INIT: {
4249 static int byte_offset = -1;
4250 static guint8 bitmask;
4253 if (byte_offset < 0)
4254 mono_marshal_find_bitfield_offset (MonoVTable, initialized, &byte_offset, &bitmask);
4256 /* Load vtable->initialized */
4257 arm_ldrsbx (code, ARMREG_IP0, sreg1, byte_offset);
4258 // FIXME: No andx_imm yet */
4259 code = mono_arm_emit_imm64 (code, ARMREG_IP1, bitmask);
4260 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
4262 arm_cbnzx (code, ARMREG_IP0, 0);
4265 g_assert (sreg1 == ARMREG_R0);
4266 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4267 (gpointer)"mono_generic_class_init");
4269 mono_arm_patch (jump, code, MONO_R_ARM64_CBZ);
4274 arm_ldrx (code, ARMREG_LR, sreg1, 0);
4277 case OP_NOT_REACHED:
4280 case OP_IL_SEQ_POINT:
4281 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
4286 case OP_COND_EXC_IC:
4287 case OP_COND_EXC_OV:
4288 case OP_COND_EXC_IOV:
4289 case OP_COND_EXC_NC:
4290 case OP_COND_EXC_INC:
4291 case OP_COND_EXC_NO:
4292 case OP_COND_EXC_INO:
4293 case OP_COND_EXC_EQ:
4294 case OP_COND_EXC_IEQ:
4295 case OP_COND_EXC_NE_UN:
4296 case OP_COND_EXC_INE_UN:
4297 case OP_COND_EXC_ILT:
4298 case OP_COND_EXC_LT:
4299 case OP_COND_EXC_ILT_UN:
4300 case OP_COND_EXC_LT_UN:
4301 case OP_COND_EXC_IGT:
4302 case OP_COND_EXC_GT:
4303 case OP_COND_EXC_IGT_UN:
4304 case OP_COND_EXC_GT_UN:
4305 case OP_COND_EXC_IGE:
4306 case OP_COND_EXC_GE:
4307 case OP_COND_EXC_IGE_UN:
4308 case OP_COND_EXC_GE_UN:
4309 case OP_COND_EXC_ILE:
4310 case OP_COND_EXC_LE:
4311 case OP_COND_EXC_ILE_UN:
4312 case OP_COND_EXC_LE_UN:
4313 code = emit_cond_exc (cfg, code, ins->opcode, ins->inst_p1);
4316 if (sreg1 != ARMREG_R0)
4317 arm_movx (code, ARMREG_R0, sreg1);
4318 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4319 (gpointer)"mono_arch_throw_exception");
4322 if (sreg1 != ARMREG_R0)
4323 arm_movx (code, ARMREG_R0, sreg1);
4324 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4325 (gpointer)"mono_arch_rethrow_exception");
4327 case OP_CALL_HANDLER:
4328 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_BL);
4330 cfg->thunk_area += THUNK_SIZE;
4332 case OP_START_HANDLER: {
4333 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4335 /* Save caller address */
4336 code = emit_strx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4339 * Reserve a param area, see test_0_finally_param_area ().
4340 * This is needed because the param area is not set up when
4341 * we are called from EH code.
4343 if (cfg->param_area)
4344 code = emit_subx_sp_imm (code, cfg->param_area);
4348 case OP_ENDFILTER: {
4349 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4351 if (cfg->param_area)
4352 code = emit_addx_sp_imm (code, cfg->param_area);
4354 if (ins->opcode == OP_ENDFILTER && sreg1 != ARMREG_R0)
4355 arm_movx (code, ARMREG_R0, sreg1);
4357 /* Return to either after the branch in OP_CALL_HANDLER, or to the EH code */
4358 code = emit_ldrx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4359 arm_brx (code, ARMREG_LR);
4363 if (ins->dreg != ARMREG_R0)
4364 arm_movx (code, ins->dreg, ARMREG_R0);
4366 case OP_GC_SAFE_POINT: {
4367 #if defined (USE_COOP_GC)
4370 arm_ldrx (code, ARMREG_IP1, ins->sreg1, 0);
4371 /* Call it if it is non-null */
4373 arm_cbzx (code, ARMREG_IP1, 0);
4374 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_threads_state_poll");
4375 mono_arm_patch (buf [0], code, MONO_R_ARM64_CBZ);
4381 g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
4382 g_assert_not_reached ();
4385 if ((cfg->opt & MONO_OPT_BRANCH) && ((code - cfg->native_code - offset) > max_len)) {
4386 g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %d)",
4387 mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
4388 g_assert_not_reached ();
4393 * If the compiled code size is larger than the bcc displacement (19 bits signed),
4394 * insert branch islands between/inside basic blocks.
4396 if (cfg->arch.cond_branch_islands)
4397 code = emit_branch_island (cfg, code, start_offset);
4399 cfg->code_len = code - cfg->native_code;
4403 emit_move_args (MonoCompile *cfg, guint8 *code)
4410 cinfo = cfg->arch.cinfo;
4412 for (i = 0; i < cinfo->nargs; ++i) {
4413 ainfo = cinfo->args + i;
4414 ins = cfg->args [i];
4416 if (ins->opcode == OP_REGVAR) {
4417 switch (ainfo->storage) {
4419 arm_movx (code, ins->dreg, ainfo->reg);
4422 switch (ainfo->slot_size) {
4425 code = emit_ldrsbx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4427 code = emit_ldrb (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4431 code = emit_ldrshx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4433 code = emit_ldrh (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4437 code = emit_ldrswx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4439 code = emit_ldrw (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4442 code = emit_ldrx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4447 g_assert_not_reached ();
4451 if (ainfo->storage != ArgVtypeByRef && ainfo->storage != ArgVtypeByRefOnStack)
4452 g_assert (ins->opcode == OP_REGOFFSET);
4454 switch (ainfo->storage) {
4456 /* Stack slots for arguments have size 8 */
4457 code = emit_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4460 code = emit_strfpx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4463 code = emit_strfpw (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4468 case ArgVtypeByRefOnStack:
4469 case ArgVtypeOnStack:
4471 case ArgVtypeByRef: {
4472 MonoInst *addr_arg = ins->inst_left;
4474 if (ainfo->gsharedvt) {
4475 g_assert (ins->opcode == OP_GSHAREDVT_ARG_REGOFFSET);
4476 arm_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4478 g_assert (ins->opcode == OP_VTARG_ADDR);
4479 g_assert (addr_arg->opcode == OP_REGOFFSET);
4480 arm_strx (code, ainfo->reg, addr_arg->inst_basereg, addr_arg->inst_offset);
4484 case ArgVtypeInIRegs:
4485 for (part = 0; part < ainfo->nregs; part ++) {
4486 code = emit_strx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + (part * 8));
4490 for (part = 0; part < ainfo->nregs; part ++) {
4491 if (ainfo->esize == 4)
4492 code = emit_strfpw (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4494 code = emit_strfpx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4498 g_assert_not_reached ();
4508 * emit_store_regarray:
4510 * Emit code to store the registers in REGS into the appropriate elements of
4511 * the register array at BASEREG+OFFSET.
4513 static __attribute__((warn_unused_result)) guint8*
4514 emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4518 for (i = 0; i < 32; ++i) {
4519 if (regs & (1 << i)) {
4520 if (i + 1 < 32 && (regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4521 arm_stpx (code, i, i + 1, basereg, offset + (i * 8));
4523 } else if (i == ARMREG_SP) {
4524 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4525 arm_strx (code, ARMREG_IP1, basereg, offset + (i * 8));
4527 arm_strx (code, i, basereg, offset + (i * 8));
4535 * emit_load_regarray:
4537 * Emit code to load the registers in REGS from the appropriate elements of
4538 * the register array at BASEREG+OFFSET.
4540 static __attribute__((warn_unused_result)) guint8*
4541 emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4545 for (i = 0; i < 32; ++i) {
4546 if (regs & (1 << i)) {
4547 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4548 if (offset + (i * 8) < 500)
4549 arm_ldpx (code, i, i + 1, basereg, offset + (i * 8));
4551 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4552 code = emit_ldrx (code, i + 1, basereg, offset + ((i + 1) * 8));
4555 } else if (i == ARMREG_SP) {
4556 g_assert_not_reached ();
4558 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4566 * emit_store_regset:
4568 * Emit code to store the registers in REGS into consecutive memory locations starting
4569 * at BASEREG+OFFSET.
4571 static __attribute__((warn_unused_result)) guint8*
4572 emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4577 for (i = 0; i < 32; ++i) {
4578 if (regs & (1 << i)) {
4579 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4580 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4583 } else if (i == ARMREG_SP) {
4584 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4585 arm_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4587 arm_strx (code, i, basereg, offset + (pos * 8));
4598 * Emit code to load the registers in REGS from consecutive memory locations starting
4599 * at BASEREG+OFFSET.
4601 static __attribute__((warn_unused_result)) guint8*
4602 emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset)
4607 for (i = 0; i < 32; ++i) {
4608 if (regs & (1 << i)) {
4609 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4610 arm_ldpx (code, i, i + 1, basereg, offset + (pos * 8));
4613 } else if (i == ARMREG_SP) {
4614 g_assert_not_reached ();
4616 arm_ldrx (code, i, basereg, offset + (pos * 8));
4624 __attribute__((warn_unused_result)) guint8*
4625 mono_arm_emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4627 return emit_load_regarray (code, regs, basereg, offset);
4630 __attribute__((warn_unused_result)) guint8*
4631 mono_arm_emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4633 return emit_store_regarray (code, regs, basereg, offset);
4636 __attribute__((warn_unused_result)) guint8*
4637 mono_arm_emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4639 return emit_store_regset (code, regs, basereg, offset);
4642 /* Same as emit_store_regset, but emit unwind info too */
4643 /* CFA_OFFSET is the offset between the CFA and basereg */
4644 static __attribute__((warn_unused_result)) guint8*
4645 emit_store_regset_cfa (MonoCompile *cfg, guint8 *code, guint64 regs, int basereg, int offset, int cfa_offset, guint64 no_cfa_regset)
4647 int i, j, pos, nregs;
4648 guint32 cfa_regset = regs & ~no_cfa_regset;
4651 for (i = 0; i < 32; ++i) {
4653 if (regs & (1 << i)) {
4654 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4656 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4658 code = emit_strx (code, i, basereg, offset + (pos * 8));
4659 code = emit_strx (code, i + 1, basereg, offset + (pos * 8) + 8);
4662 } else if (i == ARMREG_SP) {
4663 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4664 code = emit_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4666 code = emit_strx (code, i, basereg, offset + (pos * 8));
4669 for (j = 0; j < nregs; ++j) {
4670 if (cfa_regset & (1 << (i + j)))
4671 mono_emit_unwind_op_offset (cfg, code, i + j, (- cfa_offset) + offset + ((pos + j) * 8));
4684 * Emit code to initialize an LMF structure at LMF_OFFSET.
4688 emit_setup_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset, int cfa_offset)
4691 * The LMF should contain all the state required to be able to reconstruct the machine state
4692 * at the current point of execution. Since the LMF is only read during EH, only callee
4693 * saved etc. registers need to be saved.
4694 * FIXME: Save callee saved fp regs, JITted code doesn't use them, but native code does, and they
4695 * need to be restored during EH.
4699 arm_adrx (code, ARMREG_LR, code);
4700 code = emit_strx (code, ARMREG_LR, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, pc));
4701 /* gregs + fp + sp */
4702 /* Don't emit unwind info for sp/fp, they are already handled in the prolog */
4703 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_LMF_REGS, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs), cfa_offset, (1 << ARMREG_FP) | (1 << ARMREG_SP));
4709 mono_arch_emit_prolog (MonoCompile *cfg)
4711 MonoMethod *method = cfg->method;
4712 MonoMethodSignature *sig;
4715 int cfa_offset, max_offset;
4717 sig = mono_method_signature (method);
4718 cfg->code_size = 256 + sig->param_count * 64;
4719 code = cfg->native_code = g_malloc (cfg->code_size);
4721 /* This can be unaligned */
4722 cfg->stack_offset = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
4728 mono_emit_unwind_op_def_cfa (cfg, code, ARMREG_SP, 0);
4731 if (arm_is_ldpx_imm (-cfg->stack_offset)) {
4732 arm_stpx_pre (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, -cfg->stack_offset);
4734 /* sp -= cfg->stack_offset */
4735 /* This clobbers ip0/ip1 */
4736 code = emit_subx_sp_imm (code, cfg->stack_offset);
4737 arm_stpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
4739 cfa_offset += cfg->stack_offset;
4740 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
4741 mono_emit_unwind_op_offset (cfg, code, ARMREG_FP, (- cfa_offset) + 0);
4742 mono_emit_unwind_op_offset (cfg, code, ARMREG_LR, (- cfa_offset) + 8);
4743 arm_movspx (code, ARMREG_FP, ARMREG_SP);
4744 mono_emit_unwind_op_def_cfa_reg (cfg, code, ARMREG_FP);
4745 if (cfg->param_area) {
4746 /* The param area is below the frame pointer */
4747 code = emit_subx_sp_imm (code, cfg->param_area);
4750 if (cfg->method->save_lmf) {
4751 code = emit_setup_lmf (cfg, code, cfg->lmf_var->inst_offset, cfa_offset);
4754 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset, cfa_offset, 0);
4757 /* Setup args reg */
4758 if (cfg->arch.args_reg) {
4759 /* The register was already saved above */
4760 code = emit_addx_imm (code, cfg->arch.args_reg, ARMREG_FP, cfg->stack_offset);
4763 /* Save return area addr received in R8 */
4764 if (cfg->vret_addr) {
4765 MonoInst *ins = cfg->vret_addr;
4767 g_assert (ins->opcode == OP_REGOFFSET);
4768 code = emit_strx (code, ARMREG_R8, ins->inst_basereg, ins->inst_offset);
4771 /* Save mrgctx received in MONO_ARCH_RGCTX_REG */
4772 if (cfg->rgctx_var) {
4773 MonoInst *ins = cfg->rgctx_var;
4775 g_assert (ins->opcode == OP_REGOFFSET);
4777 code = emit_strx (code, MONO_ARCH_RGCTX_REG, ins->inst_basereg, ins->inst_offset);
4781 * Move arguments to their registers/stack locations.
4783 code = emit_move_args (cfg, code);
4785 /* Initialize seq_point_info_var */
4786 if (cfg->arch.seq_point_info_var) {
4787 MonoInst *ins = cfg->arch.seq_point_info_var;
4789 /* Initialize the variable from a GOT slot */
4790 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_SEQ_POINT_INFO, cfg->method);
4791 g_assert (ins->opcode == OP_REGOFFSET);
4792 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4794 /* Initialize ss_tramp_var */
4795 ins = cfg->arch.ss_tramp_var;
4796 g_assert (ins->opcode == OP_REGOFFSET);
4798 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP0, MONO_STRUCT_OFFSET (SeqPointInfo, ss_tramp_addr));
4799 code = emit_strx (code, ARMREG_IP1, ins->inst_basereg, ins->inst_offset);
4803 if (cfg->arch.ss_tramp_var) {
4804 /* Initialize ss_tramp_var */
4805 ins = cfg->arch.ss_tramp_var;
4806 g_assert (ins->opcode == OP_REGOFFSET);
4808 code = emit_imm64 (code, ARMREG_IP0, (guint64)&ss_trampoline);
4809 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4812 if (cfg->arch.bp_tramp_var) {
4813 /* Initialize bp_tramp_var */
4814 ins = cfg->arch.bp_tramp_var;
4815 g_assert (ins->opcode == OP_REGOFFSET);
4817 code = emit_imm64 (code, ARMREG_IP0, (guint64)bp_trampoline);
4818 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4823 if (cfg->opt & MONO_OPT_BRANCH) {
4824 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4826 bb->max_offset = max_offset;
4828 MONO_BB_FOR_EACH_INS (bb, ins) {
4829 max_offset += ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
4833 if (max_offset > 0x3ffff * 4)
4834 cfg->arch.cond_branch_islands = TRUE;
4840 realloc_code (MonoCompile *cfg, int size)
4842 while (cfg->code_len + size > (cfg->code_size - 16)) {
4843 cfg->code_size *= 2;
4844 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
4845 cfg->stat_code_reallocs++;
4847 return cfg->native_code + cfg->code_len;
4851 mono_arch_emit_epilog (MonoCompile *cfg)
4854 int max_epilog_size;
4858 max_epilog_size = 16 + 20*4;
4859 code = realloc_code (cfg, max_epilog_size);
4861 if (cfg->method->save_lmf) {
4862 code = mono_arm_emit_load_regarray (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->lmf_var->inst_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs) - (MONO_ARCH_FIRST_LMF_REG * 8));
4865 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4868 /* Load returned vtypes into registers if needed */
4869 cinfo = cfg->arch.cinfo;
4870 switch (cinfo->ret.storage) {
4871 case ArgVtypeInIRegs: {
4872 MonoInst *ins = cfg->ret;
4874 for (i = 0; i < cinfo->ret.nregs; ++i)
4875 code = emit_ldrx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + (i * 8));
4879 MonoInst *ins = cfg->ret;
4881 for (i = 0; i < cinfo->ret.nregs; ++i) {
4882 if (cinfo->ret.esize == 4)
4883 code = emit_ldrfpw (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
4885 code = emit_ldrfpx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
4894 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, ((1 << ARMREG_IP0) | (1 << ARMREG_IP1)));
4896 arm_retx (code, ARMREG_LR);
4898 g_assert (code - (cfg->native_code + cfg->code_len) < max_epilog_size);
4900 cfg->code_len = code - cfg->native_code;
4904 mono_arch_emit_exceptions (MonoCompile *cfg)
4907 MonoClass *exc_class;
4909 guint8* exc_throw_pos [MONO_EXC_INTRINS_NUM];
4910 guint8 exc_throw_found [MONO_EXC_INTRINS_NUM];
4911 int i, id, size = 0;
4913 for (i = 0; i < MONO_EXC_INTRINS_NUM; i++) {
4914 exc_throw_pos [i] = NULL;
4915 exc_throw_found [i] = 0;
4918 for (ji = cfg->patch_info; ji; ji = ji->next) {
4919 if (ji->type == MONO_PATCH_INFO_EXC) {
4920 i = mini_exception_id_by_name (ji->data.target);
4921 if (!exc_throw_found [i]) {
4923 exc_throw_found [i] = TRUE;
4928 code = realloc_code (cfg, size);
4930 /* Emit code to raise corlib exceptions */
4931 for (ji = cfg->patch_info; ji; ji = ji->next) {
4932 if (ji->type != MONO_PATCH_INFO_EXC)
4935 ip = cfg->native_code + ji->ip.i;
4937 id = mini_exception_id_by_name (ji->data.target);
4939 if (exc_throw_pos [id]) {
4940 /* ip points to the bcc () in OP_COND_EXC_... */
4941 arm_patch_rel (ip, exc_throw_pos [id], ji->relocation);
4942 ji->type = MONO_PATCH_INFO_NONE;
4946 exc_throw_pos [id] = code;
4947 arm_patch_rel (ip, code, ji->relocation);
4949 /* We are being branched to from the code generated by emit_cond_exc (), the pc is in ip1 */
4951 /* r0 = type token */
4952 exc_class = mono_class_load_from_name (mono_defaults.corlib, "System", ji->data.name);
4953 code = emit_imm (code, ARMREG_R0, exc_class->type_token - MONO_TOKEN_TYPE_DEF);
4955 arm_movx (code, ARMREG_R1, ARMREG_IP1);
4956 /* Branch to the corlib exception throwing trampoline */
4957 ji->ip.i = code - cfg->native_code;
4958 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
4959 ji->data.name = "mono_arch_throw_corlib_exception";
4960 ji->relocation = MONO_R_ARM64_BL;
4962 cfg->thunk_area += THUNK_SIZE;
4965 cfg->code_len = code - cfg->native_code;
4967 g_assert (cfg->code_len < cfg->code_size);
4971 mono_arch_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
4977 mono_arch_print_tree (MonoInst *tree, int arity)
4983 mono_arch_get_patch_offset (guint8 *code)
4989 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
4990 gpointer fail_tramp)
4992 int i, buf_len, imt_reg;
4996 printf ("building IMT trampoline for class %s %s entries %d code size %d code at %p end %p vtable %p\n", vtable->klass->name_space, vtable->klass->name, count, size, start, ((guint8*)start) + size, vtable);
4997 for (i = 0; i < count; ++i) {
4998 MonoIMTCheckItem *item = imt_entries [i];
4999 printf ("method %d (%p) %s vtable slot %p is_equals %d chunk size %d\n", i, item->key, item->key->name, &vtable->vtable [item->value.vtable_slot], item->is_equals, item->chunk_size);
5004 for (i = 0; i < count; ++i) {
5005 MonoIMTCheckItem *item = imt_entries [i];
5006 if (item->is_equals) {
5007 gboolean fail_case = !item->check_target_idx && fail_tramp;
5009 if (item->check_target_idx || fail_case) {
5010 if (!item->compare_done || fail_case) {
5011 buf_len += 4 * 4 + 4;
5014 if (item->has_target_code) {
5031 buf = mono_method_alloc_generic_virtual_trampoline (domain, buf_len);
5033 buf = mono_domain_code_reserve (domain, buf_len);
5037 * We are called by JITted code, which passes in the IMT argument in
5038 * MONO_ARCH_RGCTX_REG (r27). We need to preserve all caller saved regs
5041 imt_reg = MONO_ARCH_RGCTX_REG;
5042 for (i = 0; i < count; ++i) {
5043 MonoIMTCheckItem *item = imt_entries [i];
5045 item->code_target = code;
5047 if (item->is_equals) {
5049 * Check the imt argument against item->key, if equals, jump to either
5050 * item->value.target_code or to vtable [item->value.vtable_slot].
5051 * If fail_tramp is set, jump to it if not-equals.
5053 gboolean fail_case = !item->check_target_idx && fail_tramp;
5055 if (item->check_target_idx || fail_case) {
5056 /* Compare imt_reg with item->key */
5057 if (!item->compare_done || fail_case) {
5058 // FIXME: Optimize this
5059 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5060 arm_cmpx (code, imt_reg, ARMREG_IP0);
5062 item->jmp_code = code;
5063 arm_bcc (code, ARMCOND_NE, 0);
5064 /* Jump to target if equals */
5065 if (item->has_target_code) {
5066 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->value.target_code);
5067 arm_brx (code, ARMREG_IP0);
5069 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5071 code = emit_imm64 (code, ARMREG_IP0, imm);
5072 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5073 arm_brx (code, ARMREG_IP0);
5077 arm_patch_rel (item->jmp_code, code, MONO_R_ARM64_BCC);
5078 item->jmp_code = NULL;
5079 code = emit_imm64 (code, ARMREG_IP0, (guint64)fail_tramp);
5080 arm_brx (code, ARMREG_IP0);
5083 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5085 code = emit_imm64 (code, ARMREG_IP0, imm);
5086 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5087 arm_brx (code, ARMREG_IP0);
5090 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5091 arm_cmpx (code, imt_reg, ARMREG_IP0);
5092 item->jmp_code = code;
5093 arm_bcc (code, ARMCOND_HS, 0);
5096 /* Patch the branches */
5097 for (i = 0; i < count; ++i) {
5098 MonoIMTCheckItem *item = imt_entries [i];
5099 if (item->jmp_code && item->check_target_idx)
5100 arm_patch_rel (item->jmp_code, imt_entries [item->check_target_idx]->code_target, MONO_R_ARM64_BCC);
5103 g_assert ((code - buf) < buf_len);
5105 mono_arch_flush_icache (buf, code - buf);
5111 mono_arch_get_trampolines (gboolean aot)
5113 return mono_arm_get_exception_trampolines (aot);
5116 #else /* DISABLE_JIT */
5119 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
5120 gpointer fail_tramp)
5122 g_assert_not_reached ();
5126 #endif /* !DISABLE_JIT */
5128 #ifdef MONO_ARCH_SOFT_DEBUG_SUPPORTED
5131 mono_arch_set_breakpoint (MonoJitInfo *ji, guint8 *ip)
5134 guint32 native_offset = ip - (guint8*)ji->code_start;
5137 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5139 g_assert (native_offset % 4 == 0);
5140 g_assert (info->bp_addrs [native_offset / 4] == 0);
5141 info->bp_addrs [native_offset / 4] = mini_get_breakpoint_trampoline ();
5143 /* ip points to an ldrx */
5145 arm_blrx (code, ARMREG_IP0);
5146 mono_arch_flush_icache (ip, code - ip);
5151 mono_arch_clear_breakpoint (MonoJitInfo *ji, guint8 *ip)
5156 guint32 native_offset = ip - (guint8*)ji->code_start;
5157 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5159 g_assert (native_offset % 4 == 0);
5160 info->bp_addrs [native_offset / 4] = NULL;
5162 /* ip points to an ldrx */
5165 mono_arch_flush_icache (ip, code - ip);
5170 mono_arch_start_single_stepping (void)
5172 ss_trampoline = mini_get_single_step_trampoline ();
5176 mono_arch_stop_single_stepping (void)
5178 ss_trampoline = NULL;
5182 mono_arch_is_single_step_event (void *info, void *sigctx)
5184 /* We use soft breakpoints on arm64 */
5189 mono_arch_is_breakpoint_event (void *info, void *sigctx)
5191 /* We use soft breakpoints on arm64 */
5196 mono_arch_skip_breakpoint (MonoContext *ctx, MonoJitInfo *ji)
5198 g_assert_not_reached ();
5202 mono_arch_skip_single_step (MonoContext *ctx)
5204 g_assert_not_reached ();
5208 mono_arch_get_seq_point_info (MonoDomain *domain, guint8 *code)
5213 // FIXME: Add a free function
5215 mono_domain_lock (domain);
5216 info = g_hash_table_lookup (domain_jit_info (domain)->arch_seq_points,
5218 mono_domain_unlock (domain);
5221 ji = mono_jit_info_table_find (domain, (char*)code);
5224 info = g_malloc0 (sizeof (SeqPointInfo) + (ji->code_size / 4) * sizeof(guint8*));
5226 info->ss_tramp_addr = &ss_trampoline;
5228 mono_domain_lock (domain);
5229 g_hash_table_insert (domain_jit_info (domain)->arch_seq_points,
5231 mono_domain_unlock (domain);
5238 mono_arch_init_lmf_ext (MonoLMFExt *ext, gpointer prev_lmf)
5240 ext->lmf.previous_lmf = prev_lmf;
5241 /* Mark that this is a MonoLMFExt */
5242 ext->lmf.previous_lmf = (gpointer)(((gssize)ext->lmf.previous_lmf) | 2);
5243 ext->lmf.gregs [MONO_ARCH_LMF_REG_SP] = (gssize)ext;
5246 #endif /* MONO_ARCH_SOFT_DEBUG_SUPPORTED */
5249 mono_arch_opcode_supported (int opcode)
5252 case OP_ATOMIC_ADD_I4:
5253 case OP_ATOMIC_ADD_I8:
5254 case OP_ATOMIC_EXCHANGE_I4:
5255 case OP_ATOMIC_EXCHANGE_I8:
5256 case OP_ATOMIC_CAS_I4:
5257 case OP_ATOMIC_CAS_I8:
5258 case OP_ATOMIC_LOAD_I1:
5259 case OP_ATOMIC_LOAD_I2:
5260 case OP_ATOMIC_LOAD_I4:
5261 case OP_ATOMIC_LOAD_I8:
5262 case OP_ATOMIC_LOAD_U1:
5263 case OP_ATOMIC_LOAD_U2:
5264 case OP_ATOMIC_LOAD_U4:
5265 case OP_ATOMIC_LOAD_U8:
5266 case OP_ATOMIC_LOAD_R4:
5267 case OP_ATOMIC_LOAD_R8:
5268 case OP_ATOMIC_STORE_I1:
5269 case OP_ATOMIC_STORE_I2:
5270 case OP_ATOMIC_STORE_I4:
5271 case OP_ATOMIC_STORE_I8:
5272 case OP_ATOMIC_STORE_U1:
5273 case OP_ATOMIC_STORE_U2:
5274 case OP_ATOMIC_STORE_U4:
5275 case OP_ATOMIC_STORE_U8:
5276 case OP_ATOMIC_STORE_R4:
5277 case OP_ATOMIC_STORE_R8:
5285 mono_arch_get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
5287 return get_call_info (mp, sig);
5291 mono_arch_install_handler_block_guard (MonoJitInfo *ji, MonoJitExceptionInfo *clause, MonoContext *ctx, gpointer new_value)
5298 bp = MONO_CONTEXT_GET_BP (ctx);
5299 lr_loc = (gpointer*)(bp + clause->exvar_offset);
5301 old_value = *lr_loc;
5302 if ((char*)old_value < (char*)ji->code_start || (char*)old_value > ((char*)ji->code_start + ji->code_size))
5305 *lr_loc = new_value;