3 * ARM64 backend for the Mono code generator
5 * Copyright 2013 Xamarin, Inc (http://www.xamarin.com)
10 * Paolo Molaro (lupus@ximian.com)
11 * Dietmar Maurer (dietmar@ximian.com)
13 * (C) 2003 Ximian, Inc.
14 * Copyright 2003-2011 Novell, Inc (http://www.novell.com)
15 * Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
16 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
20 #include "cpu-arm64.h"
23 #include <mono/arch/arm64/arm64-codegen.h>
24 #include <mono/utils/mono-mmap.h>
25 #include <mono/utils/mono-memory-model.h>
26 #include <mono/metadata/abi-details.h>
31 * - ARM(R) Architecture Reference Manual, ARMv8, for ARMv8-A architecture profile (DDI0487A_a_armv8_arm.pdf)
32 * - Procedure Call Standard for the ARM 64-bit Architecture (AArch64) (IHI0055B_aapcs64.pdf)
33 * - ELF for the ARM 64-bit Architecture (IHI0056B_aaelf64.pdf)
36 * - ip0/ip1/lr are used as temporary registers
37 * - r27 is used as the rgctx/imt register
38 * - r28 is used to access arguments passed on the stack
39 * - d15/d16 are used as fp temporary registers
42 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
44 #define FP_TEMP_REG ARMREG_D16
45 #define FP_TEMP_REG2 ARMREG_D17
47 #define THUNK_SIZE (4 * 4)
49 /* The single step trampoline */
50 static gpointer ss_trampoline;
52 /* The breakpoint trampoline */
53 static gpointer bp_trampoline;
55 static gboolean ios_abi;
57 static __attribute__ ((__warn_unused_result__)) guint8* emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset);
60 mono_arch_regname (int reg)
62 static const char * rnames[] = {
63 "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9",
64 "r10", "r11", "r12", "r13", "r14", "r15", "r16", "r17", "r18", "r19",
65 "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", "r28", "fp",
68 if (reg >= 0 && reg < 32)
74 mono_arch_fregname (int reg)
76 static const char * rnames[] = {
77 "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9",
78 "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19",
79 "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29",
82 if (reg >= 0 && reg < 32)
88 mono_arch_get_argument_info (MonoMethodSignature *csig, int param_count, MonoJitArgumentInfo *arg_info)
94 #define MAX_ARCH_DELEGATE_PARAMS 7
97 get_delegate_invoke_impl (gboolean has_target, gboolean param_count, guint32 *code_size)
102 start = code = mono_global_codeman_reserve (12);
104 /* Replace the this argument with the target */
105 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
106 arm_ldrx (code, ARMREG_R0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, target));
107 arm_brx (code, ARMREG_IP0);
109 g_assert ((code - start) <= 12);
111 mono_arch_flush_icache (start, 12);
115 size = 8 + param_count * 4;
116 start = code = mono_global_codeman_reserve (size);
118 arm_ldrx (code, ARMREG_IP0, ARMREG_R0, MONO_STRUCT_OFFSET (MonoDelegate, method_ptr));
119 /* slide down the arguments */
120 for (i = 0; i < param_count; ++i)
121 arm_movx (code, i, i + 1);
122 arm_brx (code, ARMREG_IP0);
124 g_assert ((code - start) <= size);
126 mono_arch_flush_icache (start, size);
130 *code_size = code - start;
136 * mono_arch_get_delegate_invoke_impls:
138 * Return a list of MonoAotTrampInfo structures for the delegate invoke impl
142 mono_arch_get_delegate_invoke_impls (void)
150 code = get_delegate_invoke_impl (TRUE, 0, &code_len);
151 res = g_slist_prepend (res, mono_tramp_info_create ("delegate_invoke_impl_has_target", code, code_len, NULL, NULL));
153 for (i = 0; i <= MAX_ARCH_DELEGATE_PARAMS; ++i) {
154 code = get_delegate_invoke_impl (FALSE, i, &code_len);
155 tramp_name = g_strdup_printf ("delegate_invoke_impl_target_%d", i);
156 res = g_slist_prepend (res, mono_tramp_info_create (tramp_name, code, code_len, NULL, NULL));
164 mono_arch_get_delegate_invoke_impl (MonoMethodSignature *sig, gboolean has_target)
166 guint8 *code, *start;
169 * vtypes are returned in registers, or using the dedicated r8 register, so
170 * they can be supported by delegate invokes.
174 static guint8* cached = NULL;
180 start = mono_aot_get_trampoline ("delegate_invoke_impl_has_target");
182 start = get_delegate_invoke_impl (TRUE, 0, NULL);
183 mono_memory_barrier ();
187 static guint8* cache [MAX_ARCH_DELEGATE_PARAMS + 1] = {NULL};
190 if (sig->param_count > MAX_ARCH_DELEGATE_PARAMS)
192 for (i = 0; i < sig->param_count; ++i)
193 if (!mono_is_regsize_var (sig->params [i]))
196 code = cache [sig->param_count];
201 char *name = g_strdup_printf ("delegate_invoke_impl_target_%d", sig->param_count);
202 start = mono_aot_get_trampoline (name);
205 start = get_delegate_invoke_impl (FALSE, sig->param_count, NULL);
207 mono_memory_barrier ();
208 cache [sig->param_count] = start;
216 mono_arch_get_delegate_virtual_invoke_impl (MonoMethodSignature *sig, MonoMethod *method, int offset, gboolean load_imt_reg)
222 mono_arch_get_this_arg_from_call (mgreg_t *regs, guint8 *code)
224 return (gpointer)regs [ARMREG_R0];
228 mono_arch_cpu_init (void)
233 mono_arch_init (void)
235 mono_aot_register_jit_icall ("mono_arm_throw_exception", mono_arm_throw_exception);
236 mono_aot_register_jit_icall ("mono_arm_resume_unwind", mono_arm_resume_unwind);
239 bp_trampoline = mini_get_breakpoint_trampoline ();
241 mono_arm_gsharedvt_init ();
243 #if defined(TARGET_IOS)
249 mono_arch_cleanup (void)
254 mono_arch_cpu_optimizations (guint32 *exclude_mask)
261 mono_arch_cpu_enumerate_simd_versions (void)
267 mono_arch_register_lowlevel_calls (void)
272 mono_arch_finish_init (void)
276 /* The maximum length is 2 instructions */
278 emit_imm (guint8 *code, int dreg, int imm)
280 // FIXME: Optimize this
283 arm_movnx (code, dreg, (~limm) & 0xffff, 0);
284 arm_movkx (code, dreg, (limm >> 16) & 0xffff, 16);
286 arm_movzx (code, dreg, imm & 0xffff, 0);
288 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
294 /* The maximum length is 4 instructions */
296 emit_imm64 (guint8 *code, int dreg, guint64 imm)
298 // FIXME: Optimize this
299 arm_movzx (code, dreg, imm & 0xffff, 0);
300 if ((imm >> 16) & 0xffff)
301 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
302 if ((imm >> 32) & 0xffff)
303 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
304 if ((imm >> 48) & 0xffff)
305 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
311 mono_arm_emit_imm64 (guint8 *code, int dreg, gint64 imm)
313 return emit_imm64 (code, dreg, imm);
319 * Emit a patchable code sequence for constructing a 64 bit immediate.
322 emit_imm64_template (guint8 *code, int dreg)
324 arm_movzx (code, dreg, 0, 0);
325 arm_movkx (code, dreg, 0, 16);
326 arm_movkx (code, dreg, 0, 32);
327 arm_movkx (code, dreg, 0, 48);
332 static inline __attribute__ ((__warn_unused_result__)) guint8*
333 emit_addw_imm (guint8 *code, int dreg, int sreg, int imm)
335 if (!arm_is_arith_imm (imm)) {
336 code = emit_imm (code, ARMREG_LR, imm);
337 arm_addw (code, dreg, sreg, ARMREG_LR);
339 arm_addw_imm (code, dreg, sreg, imm);
344 static inline __attribute__ ((__warn_unused_result__)) guint8*
345 emit_addx_imm (guint8 *code, int dreg, int sreg, int imm)
347 if (!arm_is_arith_imm (imm)) {
348 code = emit_imm (code, ARMREG_LR, imm);
349 arm_addx (code, dreg, sreg, ARMREG_LR);
351 arm_addx_imm (code, dreg, sreg, imm);
356 static inline __attribute__ ((__warn_unused_result__)) guint8*
357 emit_subw_imm (guint8 *code, int dreg, int sreg, int imm)
359 if (!arm_is_arith_imm (imm)) {
360 code = emit_imm (code, ARMREG_LR, imm);
361 arm_subw (code, dreg, sreg, ARMREG_LR);
363 arm_subw_imm (code, dreg, sreg, imm);
368 static inline __attribute__ ((__warn_unused_result__)) guint8*
369 emit_subx_imm (guint8 *code, int dreg, int sreg, int imm)
371 if (!arm_is_arith_imm (imm)) {
372 code = emit_imm (code, ARMREG_LR, imm);
373 arm_subx (code, dreg, sreg, ARMREG_LR);
375 arm_subx_imm (code, dreg, sreg, imm);
380 /* Emit sp+=imm. Clobbers ip0/ip1 */
381 static inline __attribute__ ((__warn_unused_result__)) guint8*
382 emit_addx_sp_imm (guint8 *code, int imm)
384 code = emit_imm (code, ARMREG_IP0, imm);
385 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
386 arm_addx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
387 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
391 /* Emit sp-=imm. Clobbers ip0/ip1 */
392 static inline __attribute__ ((__warn_unused_result__)) guint8*
393 emit_subx_sp_imm (guint8 *code, int imm)
395 code = emit_imm (code, ARMREG_IP0, imm);
396 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
397 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
398 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
402 static inline __attribute__ ((__warn_unused_result__)) guint8*
403 emit_andw_imm (guint8 *code, int dreg, int sreg, int imm)
406 code = emit_imm (code, ARMREG_LR, imm);
407 arm_andw (code, dreg, sreg, ARMREG_LR);
412 static inline __attribute__ ((__warn_unused_result__)) guint8*
413 emit_andx_imm (guint8 *code, int dreg, int sreg, int imm)
416 code = emit_imm (code, ARMREG_LR, imm);
417 arm_andx (code, dreg, sreg, ARMREG_LR);
422 static inline __attribute__ ((__warn_unused_result__)) guint8*
423 emit_orrw_imm (guint8 *code, int dreg, int sreg, int imm)
426 code = emit_imm (code, ARMREG_LR, imm);
427 arm_orrw (code, dreg, sreg, ARMREG_LR);
432 static inline __attribute__ ((__warn_unused_result__)) guint8*
433 emit_orrx_imm (guint8 *code, int dreg, int sreg, int imm)
436 code = emit_imm (code, ARMREG_LR, imm);
437 arm_orrx (code, dreg, sreg, ARMREG_LR);
442 static inline __attribute__ ((__warn_unused_result__)) guint8*
443 emit_eorw_imm (guint8 *code, int dreg, int sreg, int imm)
446 code = emit_imm (code, ARMREG_LR, imm);
447 arm_eorw (code, dreg, sreg, ARMREG_LR);
452 static inline __attribute__ ((__warn_unused_result__)) guint8*
453 emit_eorx_imm (guint8 *code, int dreg, int sreg, int imm)
456 code = emit_imm (code, ARMREG_LR, imm);
457 arm_eorx (code, dreg, sreg, ARMREG_LR);
462 static inline __attribute__ ((__warn_unused_result__)) guint8*
463 emit_cmpw_imm (guint8 *code, int sreg, int imm)
466 arm_cmpw (code, sreg, ARMREG_RZR);
469 code = emit_imm (code, ARMREG_LR, imm);
470 arm_cmpw (code, sreg, ARMREG_LR);
476 static inline __attribute__ ((__warn_unused_result__)) guint8*
477 emit_cmpx_imm (guint8 *code, int sreg, int imm)
480 arm_cmpx (code, sreg, ARMREG_RZR);
483 code = emit_imm (code, ARMREG_LR, imm);
484 arm_cmpx (code, sreg, ARMREG_LR);
490 static inline __attribute__ ((__warn_unused_result__)) guint8*
491 emit_strb (guint8 *code, int rt, int rn, int imm)
493 if (arm_is_strb_imm (imm)) {
494 arm_strb (code, rt, rn, imm);
496 g_assert (rt != ARMREG_IP0);
497 g_assert (rn != ARMREG_IP0);
498 code = emit_imm (code, ARMREG_IP0, imm);
499 arm_strb_reg (code, rt, rn, ARMREG_IP0);
504 static inline __attribute__ ((__warn_unused_result__)) guint8*
505 emit_strh (guint8 *code, int rt, int rn, int imm)
507 if (arm_is_strh_imm (imm)) {
508 arm_strh (code, rt, rn, imm);
510 g_assert (rt != ARMREG_IP0);
511 g_assert (rn != ARMREG_IP0);
512 code = emit_imm (code, ARMREG_IP0, imm);
513 arm_strh_reg (code, rt, rn, ARMREG_IP0);
518 static inline __attribute__ ((__warn_unused_result__)) guint8*
519 emit_strw (guint8 *code, int rt, int rn, int imm)
521 if (arm_is_strw_imm (imm)) {
522 arm_strw (code, rt, rn, imm);
524 g_assert (rt != ARMREG_IP0);
525 g_assert (rn != ARMREG_IP0);
526 code = emit_imm (code, ARMREG_IP0, imm);
527 arm_strw_reg (code, rt, rn, ARMREG_IP0);
532 static inline __attribute__ ((__warn_unused_result__)) guint8*
533 emit_strfpw (guint8 *code, int rt, int rn, int imm)
535 if (arm_is_strw_imm (imm)) {
536 arm_strfpw (code, rt, rn, imm);
538 g_assert (rn != ARMREG_IP0);
539 code = emit_imm (code, ARMREG_IP0, imm);
540 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
541 arm_strfpw (code, rt, ARMREG_IP0, 0);
546 static inline __attribute__ ((__warn_unused_result__)) guint8*
547 emit_strfpx (guint8 *code, int rt, int rn, int imm)
549 if (arm_is_strx_imm (imm)) {
550 arm_strfpx (code, rt, rn, imm);
552 g_assert (rn != ARMREG_IP0);
553 code = emit_imm (code, ARMREG_IP0, imm);
554 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
555 arm_strfpx (code, rt, ARMREG_IP0, 0);
560 static inline __attribute__ ((__warn_unused_result__)) guint8*
561 emit_strx (guint8 *code, int rt, int rn, int imm)
563 if (arm_is_strx_imm (imm)) {
564 arm_strx (code, rt, rn, imm);
566 g_assert (rt != ARMREG_IP0);
567 g_assert (rn != ARMREG_IP0);
568 code = emit_imm (code, ARMREG_IP0, imm);
569 arm_strx_reg (code, rt, rn, ARMREG_IP0);
574 static inline __attribute__ ((__warn_unused_result__)) guint8*
575 emit_ldrb (guint8 *code, int rt, int rn, int imm)
577 if (arm_is_pimm12_scaled (imm, 1)) {
578 arm_ldrb (code, rt, rn, imm);
580 g_assert (rt != ARMREG_IP0);
581 g_assert (rn != ARMREG_IP0);
582 code = emit_imm (code, ARMREG_IP0, imm);
583 arm_ldrb_reg (code, rt, rn, ARMREG_IP0);
588 static inline __attribute__ ((__warn_unused_result__)) guint8*
589 emit_ldrsbx (guint8 *code, int rt, int rn, int imm)
591 if (arm_is_pimm12_scaled (imm, 1)) {
592 arm_ldrsbx (code, rt, rn, imm);
594 g_assert (rt != ARMREG_IP0);
595 g_assert (rn != ARMREG_IP0);
596 code = emit_imm (code, ARMREG_IP0, imm);
597 arm_ldrsbx_reg (code, rt, rn, ARMREG_IP0);
602 static inline __attribute__ ((__warn_unused_result__)) guint8*
603 emit_ldrh (guint8 *code, int rt, int rn, int imm)
605 if (arm_is_pimm12_scaled (imm, 2)) {
606 arm_ldrh (code, rt, rn, imm);
608 g_assert (rt != ARMREG_IP0);
609 g_assert (rn != ARMREG_IP0);
610 code = emit_imm (code, ARMREG_IP0, imm);
611 arm_ldrh_reg (code, rt, rn, ARMREG_IP0);
616 static inline __attribute__ ((__warn_unused_result__)) guint8*
617 emit_ldrshx (guint8 *code, int rt, int rn, int imm)
619 if (arm_is_pimm12_scaled (imm, 2)) {
620 arm_ldrshx (code, rt, rn, imm);
622 g_assert (rt != ARMREG_IP0);
623 g_assert (rn != ARMREG_IP0);
624 code = emit_imm (code, ARMREG_IP0, imm);
625 arm_ldrshx_reg (code, rt, rn, ARMREG_IP0);
630 static inline __attribute__ ((__warn_unused_result__)) guint8*
631 emit_ldrswx (guint8 *code, int rt, int rn, int imm)
633 if (arm_is_pimm12_scaled (imm, 4)) {
634 arm_ldrswx (code, rt, rn, imm);
636 g_assert (rt != ARMREG_IP0);
637 g_assert (rn != ARMREG_IP0);
638 code = emit_imm (code, ARMREG_IP0, imm);
639 arm_ldrswx_reg (code, rt, rn, ARMREG_IP0);
644 static inline __attribute__ ((__warn_unused_result__)) guint8*
645 emit_ldrw (guint8 *code, int rt, int rn, int imm)
647 if (arm_is_pimm12_scaled (imm, 4)) {
648 arm_ldrw (code, rt, rn, imm);
650 g_assert (rn != ARMREG_IP0);
651 code = emit_imm (code, ARMREG_IP0, imm);
652 arm_ldrw_reg (code, rt, rn, ARMREG_IP0);
657 static inline __attribute__ ((__warn_unused_result__)) guint8*
658 emit_ldrx (guint8 *code, int rt, int rn, int imm)
660 if (arm_is_pimm12_scaled (imm, 8)) {
661 arm_ldrx (code, rt, rn, imm);
663 g_assert (rn != ARMREG_IP0);
664 code = emit_imm (code, ARMREG_IP0, imm);
665 arm_ldrx_reg (code, rt, rn, ARMREG_IP0);
670 static inline __attribute__ ((__warn_unused_result__)) guint8*
671 emit_ldrfpw (guint8 *code, int rt, int rn, int imm)
673 if (arm_is_pimm12_scaled (imm, 4)) {
674 arm_ldrfpw (code, rt, rn, imm);
676 g_assert (rn != ARMREG_IP0);
677 code = emit_imm (code, ARMREG_IP0, imm);
678 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
679 arm_ldrfpw (code, rt, ARMREG_IP0, 0);
684 static inline __attribute__ ((__warn_unused_result__)) guint8*
685 emit_ldrfpx (guint8 *code, int rt, int rn, int imm)
687 if (arm_is_pimm12_scaled (imm, 8)) {
688 arm_ldrfpx (code, rt, rn, imm);
690 g_assert (rn != ARMREG_IP0);
691 code = emit_imm (code, ARMREG_IP0, imm);
692 arm_addx (code, ARMREG_IP0, rn, ARMREG_IP0);
693 arm_ldrfpx (code, rt, ARMREG_IP0, 0);
699 mono_arm_emit_ldrx (guint8 *code, int rt, int rn, int imm)
701 return emit_ldrx (code, rt, rn, imm);
705 emit_call (MonoCompile *cfg, guint8* code, guint32 patch_type, gconstpointer data)
708 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_IMM);
709 code = emit_imm64_template (code, ARMREG_LR);
710 arm_blrx (code, ARMREG_LR);
712 mono_add_patch_info_rel (cfg, code - cfg->native_code, patch_type, data, MONO_R_ARM64_BL);
714 cfg->thunk_area += THUNK_SIZE;
719 emit_aotconst_full (MonoCompile *cfg, MonoJumpInfo **ji, guint8 *code, guint8 *start, int dreg, guint32 patch_type, gconstpointer data)
722 mono_add_patch_info (cfg, code - cfg->native_code, patch_type, data);
724 *ji = mono_patch_info_list_prepend (*ji, code - start, patch_type, data);
725 /* See arch_emit_got_access () in aot-compiler.c */
726 arm_ldrx_lit (code, dreg, 0);
733 emit_aotconst (MonoCompile *cfg, guint8 *code, int dreg, guint32 patch_type, gconstpointer data)
735 return emit_aotconst_full (cfg, NULL, code, NULL, dreg, patch_type, data);
739 * mono_arm_emit_aotconst:
741 * Emit code to load an AOT constant into DREG. Usable from trampolines.
744 mono_arm_emit_aotconst (gpointer ji, guint8 *code, guint8 *code_start, int dreg, guint32 patch_type, gconstpointer data)
746 return emit_aotconst_full (NULL, (MonoJumpInfo**)ji, code, code_start, dreg, patch_type, data);
750 mono_arch_have_fast_tls (void)
760 emit_tls_get (guint8 *code, int dreg, int tls_offset)
762 arm_mrs (code, dreg, ARM_MRS_REG_TPIDR_EL0);
763 if (tls_offset < 256) {
764 arm_ldrx (code, dreg, dreg, tls_offset);
766 code = emit_addx_imm (code, dreg, dreg, tls_offset);
767 arm_ldrx (code, dreg, dreg, 0);
773 emit_tls_set (guint8 *code, int sreg, int tls_offset)
775 int tmpreg = ARMREG_IP0;
777 g_assert (sreg != tmpreg);
778 arm_mrs (code, tmpreg, ARM_MRS_REG_TPIDR_EL0);
779 if (tls_offset < 256) {
780 arm_strx (code, sreg, tmpreg, tls_offset);
782 code = emit_addx_imm (code, tmpreg, tmpreg, tls_offset);
783 arm_strx (code, sreg, tmpreg, 0);
791 * - ldrp [fp, lr], [sp], !stack_offfset
792 * Clobbers TEMP_REGS.
794 __attribute__ ((__warn_unused_result__)) guint8*
795 mono_arm_emit_destroy_frame (guint8 *code, int stack_offset, guint64 temp_regs)
797 arm_movspx (code, ARMREG_SP, ARMREG_FP);
799 if (arm_is_ldpx_imm (stack_offset)) {
800 arm_ldpx_post (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, stack_offset);
802 arm_ldpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
803 /* sp += stack_offset */
804 g_assert (temp_regs & (1 << ARMREG_IP0));
805 if (temp_regs & (1 << ARMREG_IP1)) {
806 code = emit_addx_sp_imm (code, stack_offset);
808 int imm = stack_offset;
810 /* Can't use addx_sp_imm () since we can't clobber ip0/ip1 */
811 arm_addx_imm (code, ARMREG_IP0, ARMREG_SP, 0);
813 arm_addx_imm (code, ARMREG_IP0, ARMREG_IP0, 256);
816 arm_addx_imm (code, ARMREG_SP, ARMREG_IP0, imm);
822 #define is_call_imm(diff) ((gint)(diff) >= -33554432 && (gint)(diff) <= 33554431)
825 emit_thunk (guint8 *code, gconstpointer target)
829 arm_ldrx_lit (code, ARMREG_IP0, code + 8);
830 arm_brx (code, ARMREG_IP0);
831 *(guint64*)code = (guint64)target;
832 code += sizeof (guint64);
834 mono_arch_flush_icache (p, code - p);
839 create_thunk (MonoCompile *cfg, MonoDomain *domain, guchar *code, const guchar *target)
842 MonoThunkJitInfo *info;
846 guint8 *target_thunk;
849 domain = mono_domain_get ();
853 * This can be called multiple times during JITting,
854 * save the current position in cfg->arch to avoid
855 * doing a O(n^2) search.
857 if (!cfg->arch.thunks) {
858 cfg->arch.thunks = cfg->thunks;
859 cfg->arch.thunks_size = cfg->thunk_area;
861 thunks = cfg->arch.thunks;
862 thunks_size = cfg->arch.thunks_size;
864 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, mono_method_full_name (cfg->method, TRUE));
865 g_assert_not_reached ();
868 g_assert (*(guint32*)thunks == 0);
869 emit_thunk (thunks, target);
871 cfg->arch.thunks += THUNK_SIZE;
872 cfg->arch.thunks_size -= THUNK_SIZE;
876 ji = mini_jit_info_table_find (domain, (char*)code, NULL);
878 info = mono_jit_info_get_thunk_info (ji);
881 thunks = (guint8*)ji->code_start + info->thunks_offset;
882 thunks_size = info->thunks_size;
884 orig_target = mono_arch_get_call_target (code + 4);
886 mono_domain_lock (domain);
889 if (orig_target >= thunks && orig_target < thunks + thunks_size) {
890 /* The call already points to a thunk, because of trampolines etc. */
891 target_thunk = orig_target;
893 for (p = thunks; p < thunks + thunks_size; p += THUNK_SIZE) {
894 if (((guint32*)p) [0] == 0) {
898 } else if (((guint64*)p) [1] == (guint64)target) {
899 /* Thunk already points to target */
906 //printf ("THUNK: %p %p %p\n", code, target, target_thunk);
909 mono_domain_unlock (domain);
910 g_print ("thunk failed %p->%p, thunk space=%d method %s", code, target, thunks_size, cfg ? mono_method_full_name (cfg->method, TRUE) : mono_method_full_name (jinfo_get_method (ji), TRUE));
911 g_assert_not_reached ();
914 emit_thunk (target_thunk, target);
916 mono_domain_unlock (domain);
923 arm_patch_full (MonoCompile *cfg, MonoDomain *domain, guint8 *code, guint8 *target, int relocation)
925 switch (relocation) {
927 if (arm_is_bl_disp (code, target)) {
928 arm_b (code, target);
932 thunk = create_thunk (cfg, domain, code, target);
933 g_assert (arm_is_bl_disp (code, thunk));
937 case MONO_R_ARM64_BCC: {
940 cond = arm_get_bcc_cond (code);
941 arm_bcc (code, cond, target);
944 case MONO_R_ARM64_CBZ:
945 arm_set_cbz_target (code, target);
947 case MONO_R_ARM64_IMM: {
948 guint64 imm = (guint64)target;
951 /* emit_imm64_template () */
952 dreg = arm_get_movzx_rd (code);
953 arm_movzx (code, dreg, imm & 0xffff, 0);
954 arm_movkx (code, dreg, (imm >> 16) & 0xffff, 16);
955 arm_movkx (code, dreg, (imm >> 32) & 0xffff, 32);
956 arm_movkx (code, dreg, (imm >> 48) & 0xffff, 48);
959 case MONO_R_ARM64_BL:
960 if (arm_is_bl_disp (code, target)) {
961 arm_bl (code, target);
965 thunk = create_thunk (cfg, domain, code, target);
966 g_assert (arm_is_bl_disp (code, thunk));
967 arm_bl (code, thunk);
971 g_assert_not_reached ();
976 arm_patch_rel (guint8 *code, guint8 *target, int relocation)
978 arm_patch_full (NULL, NULL, code, target, relocation);
982 mono_arm_patch (guint8 *code, guint8 *target, int relocation)
984 arm_patch_rel (code, target, relocation);
988 mono_arch_patch_code_new (MonoCompile *cfg, MonoDomain *domain, guint8 *code, MonoJumpInfo *ji, gpointer target)
992 ip = ji->ip.i + code;
995 case MONO_PATCH_INFO_METHOD_JUMP:
996 /* ji->relocation is not set by the caller */
997 arm_patch_full (cfg, domain, ip, (guint8*)target, MONO_R_ARM64_B);
1000 arm_patch_full (cfg, domain, ip, (guint8*)target, ji->relocation);
1006 mono_arch_free_jit_tls_data (MonoJitTlsData *tls)
1011 mono_arch_flush_register_windows (void)
1016 mono_arch_find_imt_method (mgreg_t *regs, guint8 *code)
1018 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1022 mono_arch_find_static_call_vtable (mgreg_t *regs, guint8 *code)
1024 return (gpointer)regs [MONO_ARCH_RGCTX_REG];
1028 mono_arch_context_get_int_reg (MonoContext *ctx, int reg)
1030 return ctx->regs [reg];
1034 mono_arch_context_set_int_reg (MonoContext *ctx, int reg, mgreg_t val)
1036 ctx->regs [reg] = val;
1040 * mono_arch_set_target:
1042 * Set the target architecture the JIT backend should generate code for, in the form
1043 * of a GNU target triplet. Only used in AOT mode.
1046 mono_arch_set_target (char *mtriple)
1048 if (strstr (mtriple, "darwin") || strstr (mtriple, "ios")) {
1054 add_general (CallInfo *cinfo, ArgInfo *ainfo, int size, gboolean sign)
1056 if (cinfo->gr >= PARAM_REGS) {
1057 ainfo->storage = ArgOnStack;
1059 /* Assume size == align */
1060 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1061 ainfo->offset = cinfo->stack_usage;
1062 ainfo->slot_size = size;
1064 cinfo->stack_usage += size;
1066 ainfo->offset = cinfo->stack_usage;
1067 ainfo->slot_size = 8;
1068 ainfo->sign = FALSE;
1069 /* Put arguments into 8 byte aligned stack slots */
1070 cinfo->stack_usage += 8;
1073 ainfo->storage = ArgInIReg;
1074 ainfo->reg = cinfo->gr;
1080 add_fp (CallInfo *cinfo, ArgInfo *ainfo, gboolean single)
1082 int size = single ? 4 : 8;
1084 if (cinfo->fr >= FP_PARAM_REGS) {
1085 ainfo->storage = single ? ArgOnStackR4 : ArgOnStackR8;
1087 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, size);
1088 ainfo->offset = cinfo->stack_usage;
1089 ainfo->slot_size = size;
1090 cinfo->stack_usage += size;
1092 ainfo->offset = cinfo->stack_usage;
1093 ainfo->slot_size = 8;
1094 /* Put arguments into 8 byte aligned stack slots */
1095 cinfo->stack_usage += 8;
1099 ainfo->storage = ArgInFRegR4;
1101 ainfo->storage = ArgInFReg;
1102 ainfo->reg = cinfo->fr;
1108 is_hfa (MonoType *t, int *out_nfields, int *out_esize, int *field_offsets)
1112 MonoClassField *field;
1113 MonoType *ftype, *prev_ftype = NULL;
1116 klass = mono_class_from_mono_type (t);
1118 while ((field = mono_class_get_fields (klass, &iter))) {
1119 if (field->type->attrs & FIELD_ATTRIBUTE_STATIC)
1121 ftype = mono_field_get_type (field);
1122 ftype = mini_get_underlying_type (ftype);
1124 if (MONO_TYPE_ISSTRUCT (ftype)) {
1125 int nested_nfields, nested_esize;
1126 int nested_field_offsets [16];
1128 if (!is_hfa (ftype, &nested_nfields, &nested_esize, nested_field_offsets))
1130 if (nested_esize == 4)
1131 ftype = &mono_defaults.single_class->byval_arg;
1133 ftype = &mono_defaults.double_class->byval_arg;
1134 if (prev_ftype && prev_ftype->type != ftype->type)
1137 for (i = 0; i < nested_nfields; ++i) {
1138 if (nfields + i < 4)
1139 field_offsets [nfields + i] = field->offset - sizeof (MonoObject) + nested_field_offsets [i];
1141 nfields += nested_nfields;
1143 if (!(!ftype->byref && (ftype->type == MONO_TYPE_R4 || ftype->type == MONO_TYPE_R8)))
1145 if (prev_ftype && prev_ftype->type != ftype->type)
1149 field_offsets [nfields] = field->offset - sizeof (MonoObject);
1153 if (nfields == 0 || nfields > 4)
1155 *out_nfields = nfields;
1156 *out_esize = prev_ftype->type == MONO_TYPE_R4 ? 4 : 8;
1161 add_valuetype (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1163 int i, size, align_size, nregs, nfields, esize;
1164 int field_offsets [16];
1167 size = mini_type_stack_size_full (t, &align, cinfo->pinvoke);
1168 align_size = ALIGN_TO (size, 8);
1170 nregs = align_size / 8;
1171 if (is_hfa (t, &nfields, &esize, field_offsets)) {
1173 * The struct might include nested float structs aligned at 8,
1174 * so need to keep track of the offsets of the individual fields.
1176 if (cinfo->fr + nfields <= FP_PARAM_REGS) {
1177 ainfo->storage = ArgHFA;
1178 ainfo->reg = cinfo->fr;
1179 ainfo->nregs = nfields;
1181 ainfo->esize = esize;
1182 for (i = 0; i < nfields; ++i)
1183 ainfo->foffsets [i] = field_offsets [i];
1184 cinfo->fr += ainfo->nregs;
1186 ainfo->nfregs_to_skip = FP_PARAM_REGS > cinfo->fr ? FP_PARAM_REGS - cinfo->fr : 0;
1187 cinfo->fr = FP_PARAM_REGS;
1188 size = ALIGN_TO (size, 8);
1189 ainfo->storage = ArgVtypeOnStack;
1190 ainfo->offset = cinfo->stack_usage;
1193 ainfo->nregs = nfields;
1194 ainfo->esize = esize;
1195 cinfo->stack_usage += size;
1200 if (align_size > 16) {
1201 ainfo->storage = ArgVtypeByRef;
1206 if (cinfo->gr + nregs > PARAM_REGS) {
1207 size = ALIGN_TO (size, 8);
1208 ainfo->storage = ArgVtypeOnStack;
1209 ainfo->offset = cinfo->stack_usage;
1211 cinfo->stack_usage += size;
1212 cinfo->gr = PARAM_REGS;
1214 ainfo->storage = ArgVtypeInIRegs;
1215 ainfo->reg = cinfo->gr;
1216 ainfo->nregs = nregs;
1223 add_param (CallInfo *cinfo, ArgInfo *ainfo, MonoType *t)
1227 ptype = mini_get_underlying_type (t);
1228 switch (ptype->type) {
1230 add_general (cinfo, ainfo, 1, TRUE);
1233 add_general (cinfo, ainfo, 1, FALSE);
1236 add_general (cinfo, ainfo, 2, TRUE);
1239 add_general (cinfo, ainfo, 2, FALSE);
1242 add_general (cinfo, ainfo, 4, TRUE);
1245 add_general (cinfo, ainfo, 4, FALSE);
1250 case MONO_TYPE_FNPTR:
1251 case MONO_TYPE_OBJECT:
1254 add_general (cinfo, ainfo, 8, FALSE);
1257 add_fp (cinfo, ainfo, FALSE);
1260 add_fp (cinfo, ainfo, TRUE);
1262 case MONO_TYPE_VALUETYPE:
1263 case MONO_TYPE_TYPEDBYREF:
1264 add_valuetype (cinfo, ainfo, ptype);
1266 case MONO_TYPE_VOID:
1267 ainfo->storage = ArgNone;
1269 case MONO_TYPE_GENERICINST:
1270 if (!mono_type_generic_inst_is_valuetype (ptype)) {
1271 add_general (cinfo, ainfo, 8, FALSE);
1272 } else if (mini_is_gsharedvt_variable_type (ptype)) {
1274 * Treat gsharedvt arguments as large vtypes
1276 ainfo->storage = ArgVtypeByRef;
1277 ainfo->gsharedvt = TRUE;
1279 add_valuetype (cinfo, ainfo, ptype);
1283 case MONO_TYPE_MVAR:
1284 g_assert (mini_is_gsharedvt_type (ptype));
1285 ainfo->storage = ArgVtypeByRef;
1286 ainfo->gsharedvt = TRUE;
1289 g_assert_not_reached ();
1297 * Obtain information about a call according to the calling convention.
1300 get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
1304 int n, pstart, pindex;
1306 n = sig->hasthis + sig->param_count;
1309 cinfo = mono_mempool_alloc0 (mp, sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1311 cinfo = g_malloc0 (sizeof (CallInfo) + (sizeof (ArgInfo) * n));
1314 cinfo->pinvoke = sig->pinvoke;
1317 add_param (cinfo, &cinfo->ret, sig->ret);
1318 if (cinfo->ret.storage == ArgVtypeByRef)
1319 cinfo->ret.reg = ARMREG_R8;
1323 cinfo->stack_usage = 0;
1327 add_general (cinfo, cinfo->args + 0, 8, FALSE);
1329 for (pindex = pstart; pindex < sig->param_count; ++pindex) {
1330 ainfo = cinfo->args + sig->hasthis + pindex;
1332 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1333 /* Prevent implicit arguments and sig_cookie from
1334 being passed in registers */
1335 cinfo->gr = PARAM_REGS;
1336 cinfo->fr = FP_PARAM_REGS;
1337 /* Emit the signature cookie just before the implicit arguments */
1338 add_param (cinfo, &cinfo->sig_cookie, &mono_defaults.int_class->byval_arg);
1341 add_param (cinfo, ainfo, sig->params [pindex]);
1342 if (ainfo->storage == ArgVtypeByRef) {
1343 /* Pass the argument address in the next register */
1344 if (cinfo->gr >= PARAM_REGS) {
1345 ainfo->storage = ArgVtypeByRefOnStack;
1346 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, 8);
1347 ainfo->offset = cinfo->stack_usage;
1348 cinfo->stack_usage += 8;
1350 ainfo->reg = cinfo->gr;
1356 /* Handle the case where there are no implicit arguments */
1357 if ((sig->call_convention == MONO_CALL_VARARG) && (pindex == sig->sentinelpos)) {
1358 /* Prevent implicit arguments and sig_cookie from
1359 being passed in registers */
1360 cinfo->gr = PARAM_REGS;
1361 cinfo->fr = FP_PARAM_REGS;
1362 /* Emit the signature cookie just before the implicit arguments */
1363 add_param (cinfo, &cinfo->sig_cookie, &mono_defaults.int_class->byval_arg);
1366 cinfo->stack_usage = ALIGN_TO (cinfo->stack_usage, MONO_ARCH_FRAME_ALIGNMENT);
1372 MonoMethodSignature *sig;
1375 MonoType **param_types;
1376 int n_fpargs, n_fpret;
1380 dyn_call_supported (CallInfo *cinfo, MonoMethodSignature *sig)
1384 // FIXME: Add more cases
1385 switch (cinfo->ret.storage) {
1392 case ArgVtypeInIRegs:
1393 if (cinfo->ret.nregs > 2)
1402 for (i = 0; i < cinfo->nargs; ++i) {
1403 ArgInfo *ainfo = &cinfo->args [i];
1405 switch (ainfo->storage) {
1407 case ArgVtypeInIRegs:
1423 mono_arch_dyn_call_prepare (MonoMethodSignature *sig)
1425 ArchDynCallInfo *info;
1429 cinfo = get_call_info (NULL, sig);
1431 if (!dyn_call_supported (cinfo, sig)) {
1436 info = g_new0 (ArchDynCallInfo, 1);
1437 // FIXME: Preprocess the info to speed up start_dyn_call ()
1439 info->cinfo = cinfo;
1440 info->rtype = mini_get_underlying_type (sig->ret);
1441 info->param_types = g_new0 (MonoType*, sig->param_count);
1442 for (i = 0; i < sig->param_count; ++i)
1443 info->param_types [i] = mini_get_underlying_type (sig->params [i]);
1445 switch (cinfo->ret.storage) {
1451 info->n_fpret = cinfo->ret.nregs;
1457 return (MonoDynCallInfo*)info;
1461 mono_arch_dyn_call_free (MonoDynCallInfo *info)
1463 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1465 g_free (ainfo->cinfo);
1466 g_free (ainfo->param_types);
1471 mono_arch_dyn_call_get_buf_size (MonoDynCallInfo *info)
1473 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1475 g_assert (ainfo->cinfo->stack_usage % MONO_ARCH_FRAME_ALIGNMENT == 0);
1476 return sizeof (DynCallArgs) + ainfo->cinfo->stack_usage;
1480 bitcast_r4_to_r8 (float f)
1488 bitcast_r8_to_r4 (double f)
1496 mono_arch_start_dyn_call (MonoDynCallInfo *info, gpointer **args, guint8 *ret, guint8 *buf)
1498 ArchDynCallInfo *dinfo = (ArchDynCallInfo*)info;
1499 DynCallArgs *p = (DynCallArgs*)buf;
1500 int aindex, arg_index, greg, i, pindex;
1501 MonoMethodSignature *sig = dinfo->sig;
1502 CallInfo *cinfo = dinfo->cinfo;
1503 int buffer_offset = 0;
1507 p->n_fpargs = dinfo->n_fpargs;
1508 p->n_fpret = dinfo->n_fpret;
1509 p->n_stackargs = cinfo->stack_usage / sizeof (mgreg_t);
1516 p->regs [greg ++] = (mgreg_t)*(args [arg_index ++]);
1518 if (cinfo->ret.storage == ArgVtypeByRef)
1519 p->regs [ARMREG_R8] = (mgreg_t)ret;
1521 for (aindex = pindex; aindex < sig->param_count; aindex++) {
1522 MonoType *t = dinfo->param_types [aindex];
1523 gpointer *arg = args [arg_index ++];
1524 ArgInfo *ainfo = &cinfo->args [aindex + sig->hasthis];
1527 if (ainfo->storage == ArgOnStack) {
1528 slot = PARAM_REGS + 1 + (ainfo->offset / sizeof (mgreg_t));
1534 p->regs [slot] = (mgreg_t)*arg;
1538 if (ios_abi && ainfo->storage == ArgOnStack) {
1539 guint8 *stack_arg = (guint8*)&(p->regs [PARAM_REGS + 1]) + ainfo->offset;
1540 gboolean handled = TRUE;
1542 /* Special case arguments smaller than 1 machine word */
1545 *(guint8*)stack_arg = *(guint8*)arg;
1548 *(gint8*)stack_arg = *(gint8*)arg;
1551 *(guint16*)stack_arg = *(guint16*)arg;
1554 *(gint16*)stack_arg = *(gint16*)arg;
1557 *(gint32*)stack_arg = *(gint32*)arg;
1560 *(guint32*)stack_arg = *(guint32*)arg;
1571 case MONO_TYPE_OBJECT:
1577 p->regs [slot] = (mgreg_t)*arg;
1580 p->regs [slot] = *(guint8*)arg;
1583 p->regs [slot] = *(gint8*)arg;
1586 p->regs [slot] = *(gint16*)arg;
1589 p->regs [slot] = *(guint16*)arg;
1592 p->regs [slot] = *(gint32*)arg;
1595 p->regs [slot] = *(guint32*)arg;
1598 p->fpregs [ainfo->reg] = bitcast_r4_to_r8 (*(float*)arg);
1602 p->fpregs [ainfo->reg] = *(double*)arg;
1605 case MONO_TYPE_GENERICINST:
1606 if (MONO_TYPE_IS_REFERENCE (t)) {
1607 p->regs [slot] = (mgreg_t)*arg;
1610 if (t->type == MONO_TYPE_GENERICINST && mono_class_is_nullable (mono_class_from_mono_type (t))) {
1611 MonoClass *klass = mono_class_from_mono_type (t);
1612 guint8 *nullable_buf;
1616 * Use p->buffer as a temporary buffer since the data needs to be available after this call
1617 * if the nullable param is passed by ref.
1619 size = mono_class_value_size (klass, NULL);
1620 nullable_buf = p->buffer + buffer_offset;
1621 buffer_offset += size;
1622 g_assert (buffer_offset <= 256);
1624 /* The argument pointed to by arg is either a boxed vtype or null */
1625 mono_nullable_init (nullable_buf, (MonoObject*)arg, klass);
1627 arg = (gpointer*)nullable_buf;
1633 case MONO_TYPE_VALUETYPE:
1634 switch (ainfo->storage) {
1635 case ArgVtypeInIRegs:
1636 for (i = 0; i < ainfo->nregs; ++i)
1637 p->regs [slot ++] = ((mgreg_t*)arg) [i];
1640 if (ainfo->esize == 4) {
1641 for (i = 0; i < ainfo->nregs; ++i)
1642 p->fpregs [ainfo->reg + i] = bitcast_r4_to_r8 (((float*)arg) [ainfo->foffsets [i] / 4]);
1644 for (i = 0; i < ainfo->nregs; ++i)
1645 p->fpregs [ainfo->reg + i] = ((double*)arg) [ainfo->foffsets [i] / 8];
1647 p->n_fpargs += ainfo->nregs;
1650 p->regs [slot] = (mgreg_t)arg;
1653 g_assert_not_reached ();
1658 g_assert_not_reached ();
1664 mono_arch_finish_dyn_call (MonoDynCallInfo *info, guint8 *buf)
1666 ArchDynCallInfo *ainfo = (ArchDynCallInfo*)info;
1667 CallInfo *cinfo = ainfo->cinfo;
1668 DynCallArgs *args = (DynCallArgs*)buf;
1669 MonoType *ptype = ainfo->rtype;
1670 guint8 *ret = args->ret;
1671 mgreg_t res = args->res;
1672 mgreg_t res2 = args->res2;
1675 if (cinfo->ret.storage == ArgVtypeByRef)
1678 switch (ptype->type) {
1679 case MONO_TYPE_VOID:
1680 *(gpointer*)ret = NULL;
1682 case MONO_TYPE_OBJECT:
1686 *(gpointer*)ret = (gpointer)res;
1692 *(guint8*)ret = res;
1695 *(gint16*)ret = res;
1698 *(guint16*)ret = res;
1701 *(gint32*)ret = res;
1704 *(guint32*)ret = res;
1708 *(guint64*)ret = res;
1711 *(float*)ret = bitcast_r8_to_r4 (args->fpregs [0]);
1714 *(double*)ret = args->fpregs [0];
1716 case MONO_TYPE_GENERICINST:
1717 if (MONO_TYPE_IS_REFERENCE (ptype)) {
1718 *(gpointer*)ret = (gpointer)res;
1723 case MONO_TYPE_VALUETYPE:
1724 switch (ainfo->cinfo->ret.storage) {
1725 case ArgVtypeInIRegs:
1726 *(mgreg_t*)ret = res;
1727 if (ainfo->cinfo->ret.nregs > 1)
1728 ((mgreg_t*)ret) [1] = res2;
1731 /* Use the same area for returning fp values */
1732 if (cinfo->ret.esize == 4) {
1733 for (i = 0; i < cinfo->ret.nregs; ++i)
1734 ((float*)ret) [cinfo->ret.foffsets [i] / 4] = bitcast_r8_to_r4 (args->fpregs [i]);
1736 for (i = 0; i < cinfo->ret.nregs; ++i)
1737 ((double*)ret) [cinfo->ret.foffsets [i] / 8] = args->fpregs [i];
1741 g_assert_not_reached ();
1746 g_assert_not_reached ();
1751 void sys_icache_invalidate (void *start, size_t len);
1755 mono_arch_flush_icache (guint8 *code, gint size)
1757 #ifndef MONO_CROSS_COMPILE
1759 sys_icache_invalidate (code, size);
1761 /* Don't rely on GCC's __clear_cache implementation, as it caches
1762 * icache/dcache cache line sizes, that can vary between cores on
1763 * big.LITTLE architectures. */
1764 guint64 end = (guint64) (code + size);
1766 /* always go with cacheline size of 4 bytes as this code isn't perf critical
1767 * anyway. Reading the cache line size from a machine register can be racy
1768 * on a big.LITTLE architecture if the cores don't have the same cache line
1770 const size_t icache_line_size = 4;
1771 const size_t dcache_line_size = 4;
1773 addr = (guint64) code & ~(guint64) (dcache_line_size - 1);
1774 for (; addr < end; addr += dcache_line_size)
1775 asm volatile("dc civac, %0" : : "r" (addr) : "memory");
1776 asm volatile("dsb ish" : : : "memory");
1778 addr = (guint64) code & ~(guint64) (icache_line_size - 1);
1779 for (; addr < end; addr += icache_line_size)
1780 asm volatile("ic ivau, %0" : : "r" (addr) : "memory");
1782 asm volatile ("dsb ish" : : : "memory");
1783 asm volatile ("isb" : : : "memory");
1791 mono_arch_opcode_needs_emulation (MonoCompile *cfg, int opcode)
1798 mono_arch_get_allocatable_int_vars (MonoCompile *cfg)
1803 for (i = 0; i < cfg->num_varinfo; i++) {
1804 MonoInst *ins = cfg->varinfo [i];
1805 MonoMethodVar *vmv = MONO_VARINFO (cfg, i);
1808 if (vmv->range.first_use.abs_pos >= vmv->range.last_use.abs_pos)
1811 if ((ins->flags & (MONO_INST_IS_DEAD|MONO_INST_VOLATILE|MONO_INST_INDIRECT)) ||
1812 (ins->opcode != OP_LOCAL && ins->opcode != OP_ARG))
1815 if (mono_is_regsize_var (ins->inst_vtype)) {
1816 g_assert (MONO_VARINFO (cfg, i)->reg == -1);
1817 g_assert (i == vmv->idx);
1818 vars = g_list_prepend (vars, vmv);
1822 vars = mono_varlist_sort (cfg, vars, 0);
1828 mono_arch_get_global_int_regs (MonoCompile *cfg)
1833 /* r28 is reserved for cfg->arch.args_reg */
1834 /* r27 is reserved for the imt argument */
1835 for (i = ARMREG_R19; i <= ARMREG_R26; ++i)
1836 regs = g_list_prepend (regs, GUINT_TO_POINTER (i));
1842 mono_arch_regalloc_cost (MonoCompile *cfg, MonoMethodVar *vmv)
1844 MonoInst *ins = cfg->varinfo [vmv->idx];
1846 if (ins->opcode == OP_ARG)
1853 mono_arch_create_vars (MonoCompile *cfg)
1855 MonoMethodSignature *sig;
1858 sig = mono_method_signature (cfg->method);
1859 if (!cfg->arch.cinfo)
1860 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
1861 cinfo = cfg->arch.cinfo;
1863 if (cinfo->ret.storage == ArgVtypeByRef) {
1864 cfg->vret_addr = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1865 cfg->vret_addr->flags |= MONO_INST_VOLATILE;
1868 if (cfg->gen_sdb_seq_points) {
1871 if (cfg->compile_aot) {
1872 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1873 ins->flags |= MONO_INST_VOLATILE;
1874 cfg->arch.seq_point_info_var = ins;
1877 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1878 ins->flags |= MONO_INST_VOLATILE;
1879 cfg->arch.ss_tramp_var = ins;
1881 ins = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
1882 ins->flags |= MONO_INST_VOLATILE;
1883 cfg->arch.bp_tramp_var = ins;
1886 if (cfg->method->save_lmf) {
1887 cfg->create_lmf_var = TRUE;
1893 mono_arch_allocate_vars (MonoCompile *cfg)
1895 MonoMethodSignature *sig;
1899 int i, offset, size, align;
1900 guint32 locals_stack_size, locals_stack_align;
1904 * Allocate arguments and locals to either register (OP_REGVAR) or to a stack slot (OP_REGOFFSET).
1905 * Compute cfg->stack_offset and update cfg->used_int_regs.
1908 sig = mono_method_signature (cfg->method);
1910 if (!cfg->arch.cinfo)
1911 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
1912 cinfo = cfg->arch.cinfo;
1915 * The ARM64 ABI always uses a frame pointer.
1916 * The instruction set prefers positive offsets, so fp points to the bottom of the
1917 * frame, and stack slots are at positive offsets.
1918 * If some arguments are received on the stack, their offsets relative to fp can
1919 * not be computed right now because the stack frame might grow due to spilling
1920 * done by the local register allocator. To solve this, we reserve a register
1921 * which points to them.
1922 * The stack frame looks like this:
1923 * args_reg -> <bottom of parent frame>
1925 * fp -> <saved fp+lr>
1926 * sp -> <localloc/params area>
1928 cfg->frame_reg = ARMREG_FP;
1929 cfg->flags |= MONO_CFG_HAS_SPILLUP;
1935 if (cinfo->stack_usage) {
1936 g_assert (!(cfg->used_int_regs & (1 << ARMREG_R28)));
1937 cfg->arch.args_reg = ARMREG_R28;
1938 cfg->used_int_regs |= 1 << ARMREG_R28;
1941 if (cfg->method->save_lmf) {
1942 /* The LMF var is allocated normally */
1944 /* Callee saved regs */
1945 cfg->arch.saved_gregs_offset = offset;
1946 for (i = 0; i < 32; ++i)
1947 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) && (cfg->used_int_regs & (1 << i)))
1952 switch (cinfo->ret.storage) {
1958 cfg->ret->opcode = OP_REGVAR;
1959 cfg->ret->dreg = cinfo->ret.reg;
1961 case ArgVtypeInIRegs:
1963 /* Allocate a local to hold the result, the epilog will copy it to the correct place */
1964 cfg->ret->opcode = OP_REGOFFSET;
1965 cfg->ret->inst_basereg = cfg->frame_reg;
1966 cfg->ret->inst_offset = offset;
1967 if (cinfo->ret.storage == ArgHFA)
1974 /* This variable will be initalized in the prolog from R8 */
1975 cfg->vret_addr->opcode = OP_REGOFFSET;
1976 cfg->vret_addr->inst_basereg = cfg->frame_reg;
1977 cfg->vret_addr->inst_offset = offset;
1979 if (G_UNLIKELY (cfg->verbose_level > 1)) {
1980 printf ("vret_addr =");
1981 mono_print_ins (cfg->vret_addr);
1985 g_assert_not_reached ();
1990 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
1991 ainfo = cinfo->args + i;
1993 ins = cfg->args [i];
1994 if (ins->opcode == OP_REGVAR)
1997 ins->opcode = OP_REGOFFSET;
1998 ins->inst_basereg = cfg->frame_reg;
2000 switch (ainfo->storage) {
2004 // FIXME: Use nregs/size
2005 /* These will be copied to the stack in the prolog */
2006 ins->inst_offset = offset;
2012 case ArgVtypeOnStack:
2013 /* These are in the parent frame */
2014 g_assert (cfg->arch.args_reg);
2015 ins->inst_basereg = cfg->arch.args_reg;
2016 ins->inst_offset = ainfo->offset;
2018 case ArgVtypeInIRegs:
2020 ins->opcode = OP_REGOFFSET;
2021 ins->inst_basereg = cfg->frame_reg;
2022 /* These arguments are saved to the stack in the prolog */
2023 ins->inst_offset = offset;
2024 if (cfg->verbose_level >= 2)
2025 printf ("arg %d allocated to %s+0x%0x.\n", i, mono_arch_regname (ins->inst_basereg), (int)ins->inst_offset);
2026 if (ainfo->storage == ArgHFA)
2032 case ArgVtypeByRefOnStack: {
2035 if (ainfo->gsharedvt) {
2036 ins->opcode = OP_REGOFFSET;
2037 ins->inst_basereg = cfg->arch.args_reg;
2038 ins->inst_offset = ainfo->offset;
2042 /* The vtype address is in the parent frame */
2043 g_assert (cfg->arch.args_reg);
2044 MONO_INST_NEW (cfg, vtaddr, 0);
2045 vtaddr->opcode = OP_REGOFFSET;
2046 vtaddr->inst_basereg = cfg->arch.args_reg;
2047 vtaddr->inst_offset = ainfo->offset;
2049 /* Need an indirection */
2050 ins->opcode = OP_VTARG_ADDR;
2051 ins->inst_left = vtaddr;
2054 case ArgVtypeByRef: {
2057 if (ainfo->gsharedvt) {
2058 ins->opcode = OP_REGOFFSET;
2059 ins->inst_basereg = cfg->frame_reg;
2060 ins->inst_offset = offset;
2065 /* The vtype address is in a register, will be copied to the stack in the prolog */
2066 MONO_INST_NEW (cfg, vtaddr, 0);
2067 vtaddr->opcode = OP_REGOFFSET;
2068 vtaddr->inst_basereg = cfg->frame_reg;
2069 vtaddr->inst_offset = offset;
2072 /* Need an indirection */
2073 ins->opcode = OP_VTARG_ADDR;
2074 ins->inst_left = vtaddr;
2078 g_assert_not_reached ();
2083 /* Allocate these first so they have a small offset, OP_SEQ_POINT depends on this */
2084 // FIXME: Allocate these to registers
2085 ins = cfg->arch.seq_point_info_var;
2089 offset += align - 1;
2090 offset &= ~(align - 1);
2091 ins->opcode = OP_REGOFFSET;
2092 ins->inst_basereg = cfg->frame_reg;
2093 ins->inst_offset = offset;
2096 ins = cfg->arch.ss_tramp_var;
2100 offset += align - 1;
2101 offset &= ~(align - 1);
2102 ins->opcode = OP_REGOFFSET;
2103 ins->inst_basereg = cfg->frame_reg;
2104 ins->inst_offset = offset;
2107 ins = cfg->arch.bp_tramp_var;
2111 offset += align - 1;
2112 offset &= ~(align - 1);
2113 ins->opcode = OP_REGOFFSET;
2114 ins->inst_basereg = cfg->frame_reg;
2115 ins->inst_offset = offset;
2120 offsets = mono_allocate_stack_slots (cfg, FALSE, &locals_stack_size, &locals_stack_align);
2121 if (locals_stack_align)
2122 offset = ALIGN_TO (offset, locals_stack_align);
2124 for (i = cfg->locals_start; i < cfg->num_varinfo; i++) {
2125 if (offsets [i] != -1) {
2126 ins = cfg->varinfo [i];
2127 ins->opcode = OP_REGOFFSET;
2128 ins->inst_basereg = cfg->frame_reg;
2129 ins->inst_offset = offset + offsets [i];
2130 //printf ("allocated local %d to ", i); mono_print_tree_nl (ins);
2133 offset += locals_stack_size;
2135 offset = ALIGN_TO (offset, MONO_ARCH_FRAME_ALIGNMENT);
2137 cfg->stack_offset = offset;
2142 mono_arch_get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
2147 LLVMCallInfo *linfo;
2149 n = sig->param_count + sig->hasthis;
2151 cinfo = get_call_info (cfg->mempool, sig);
2153 linfo = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
2155 switch (cinfo->ret.storage) {
2162 linfo->ret.storage = LLVMArgVtypeByRef;
2165 // FIXME: This doesn't work yet since the llvm backend represents these types as an i8
2166 // array which is returned in int regs
2169 linfo->ret.storage = LLVMArgFpStruct;
2170 linfo->ret.nslots = cinfo->ret.nregs;
2171 linfo->ret.esize = cinfo->ret.esize;
2173 case ArgVtypeInIRegs:
2174 /* LLVM models this by returning an int */
2175 linfo->ret.storage = LLVMArgVtypeAsScalar;
2176 linfo->ret.nslots = cinfo->ret.nregs;
2177 linfo->ret.esize = cinfo->ret.esize;
2180 g_assert_not_reached ();
2184 for (i = 0; i < n; ++i) {
2185 LLVMArgInfo *lainfo = &linfo->args [i];
2187 ainfo = cinfo->args + i;
2189 lainfo->storage = LLVMArgNone;
2191 switch (ainfo->storage) {
2198 lainfo->storage = LLVMArgNormal;
2201 case ArgVtypeByRefOnStack:
2202 lainfo->storage = LLVMArgVtypeByRef;
2207 lainfo->storage = LLVMArgAsFpArgs;
2208 lainfo->nslots = ainfo->nregs;
2209 lainfo->esize = ainfo->esize;
2210 for (j = 0; j < ainfo->nregs; ++j)
2211 lainfo->pair_storage [j] = LLVMArgInFPReg;
2214 case ArgVtypeInIRegs:
2215 lainfo->storage = LLVMArgAsIArgs;
2216 lainfo->nslots = ainfo->nregs;
2218 case ArgVtypeOnStack:
2222 lainfo->storage = LLVMArgAsFpArgs;
2223 lainfo->nslots = ainfo->nregs;
2224 lainfo->esize = ainfo->esize;
2225 lainfo->ndummy_fpargs = ainfo->nfregs_to_skip;
2226 for (j = 0; j < ainfo->nregs; ++j)
2227 lainfo->pair_storage [j] = LLVMArgInFPReg;
2229 lainfo->storage = LLVMArgAsIArgs;
2230 lainfo->nslots = ainfo->size / 8;
2234 g_assert_not_reached ();
2244 add_outarg_reg (MonoCompile *cfg, MonoCallInst *call, ArgStorage storage, int reg, MonoInst *arg)
2250 MONO_INST_NEW (cfg, ins, OP_MOVE);
2251 ins->dreg = mono_alloc_ireg_copy (cfg, arg->dreg);
2252 ins->sreg1 = arg->dreg;
2253 MONO_ADD_INS (cfg->cbb, ins);
2254 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, FALSE);
2257 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2258 ins->dreg = mono_alloc_freg (cfg);
2259 ins->sreg1 = arg->dreg;
2260 MONO_ADD_INS (cfg->cbb, ins);
2261 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2264 if (COMPILE_LLVM (cfg))
2265 MONO_INST_NEW (cfg, ins, OP_FMOVE);
2267 MONO_INST_NEW (cfg, ins, OP_RMOVE);
2269 MONO_INST_NEW (cfg, ins, OP_ARM_SETFREG_R4);
2270 ins->dreg = mono_alloc_freg (cfg);
2271 ins->sreg1 = arg->dreg;
2272 MONO_ADD_INS (cfg->cbb, ins);
2273 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, reg, TRUE);
2276 g_assert_not_reached ();
2282 emit_sig_cookie (MonoCompile *cfg, MonoCallInst *call, CallInfo *cinfo)
2284 MonoMethodSignature *tmp_sig;
2287 if (call->tail_call)
2290 g_assert (cinfo->sig_cookie.storage == ArgOnStack);
2293 * mono_ArgIterator_Setup assumes the signature cookie is
2294 * passed first and all the arguments which were before it are
2295 * passed on the stack after the signature. So compensate by
2296 * passing a different signature.
2298 tmp_sig = mono_metadata_signature_dup (call->signature);
2299 tmp_sig->param_count -= call->signature->sentinelpos;
2300 tmp_sig->sentinelpos = 0;
2301 memcpy (tmp_sig->params, call->signature->params + call->signature->sentinelpos, tmp_sig->param_count * sizeof (MonoType*));
2303 sig_reg = mono_alloc_ireg (cfg);
2304 MONO_EMIT_NEW_SIGNATURECONST (cfg, sig_reg, tmp_sig);
2306 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, cinfo->sig_cookie.offset, sig_reg);
2310 mono_arch_emit_call (MonoCompile *cfg, MonoCallInst *call)
2312 MonoMethodSignature *sig;
2313 MonoInst *arg, *vtarg;
2318 sig = call->signature;
2320 cinfo = get_call_info (cfg->mempool, sig);
2322 switch (cinfo->ret.storage) {
2323 case ArgVtypeInIRegs:
2326 * The vtype is returned in registers, save the return area address in a local, and save the vtype into
2327 * the location pointed to by it after call in emit_move_return_value ().
2329 if (!cfg->arch.vret_addr_loc) {
2330 cfg->arch.vret_addr_loc = mono_compile_create_var (cfg, &mono_defaults.int_class->byval_arg, OP_LOCAL);
2331 /* Prevent it from being register allocated or optimized away */
2332 ((MonoInst*)cfg->arch.vret_addr_loc)->flags |= MONO_INST_VOLATILE;
2335 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, ((MonoInst*)cfg->arch.vret_addr_loc)->dreg, call->vret_var->dreg);
2338 /* Pass the vtype return address in R8 */
2339 MONO_INST_NEW (cfg, vtarg, OP_MOVE);
2340 vtarg->sreg1 = call->vret_var->dreg;
2341 vtarg->dreg = mono_alloc_preg (cfg);
2342 MONO_ADD_INS (cfg->cbb, vtarg);
2344 mono_call_inst_add_outarg_reg (cfg, call, vtarg->dreg, cinfo->ret.reg, FALSE);
2350 for (i = 0; i < cinfo->nargs; ++i) {
2351 ainfo = cinfo->args + i;
2352 arg = call->args [i];
2354 if ((sig->call_convention == MONO_CALL_VARARG) && (i == sig->sentinelpos)) {
2355 /* Emit the signature cookie just before the implicit arguments */
2356 emit_sig_cookie (cfg, call, cinfo);
2359 switch (ainfo->storage) {
2363 add_outarg_reg (cfg, call, ainfo->storage, ainfo->reg, arg);
2366 switch (ainfo->slot_size) {
2368 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2371 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2374 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2377 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2380 g_assert_not_reached ();
2385 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER8_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2388 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORER4_MEMBASE_REG, ARMREG_SP, ainfo->offset, arg->dreg);
2390 case ArgVtypeInIRegs:
2392 case ArgVtypeByRefOnStack:
2393 case ArgVtypeOnStack:
2399 size = mono_class_value_size (arg->klass, &align);
2401 MONO_INST_NEW (cfg, ins, OP_OUTARG_VT);
2402 ins->sreg1 = arg->dreg;
2403 ins->klass = arg->klass;
2404 ins->backend.size = size;
2405 ins->inst_p0 = call;
2406 ins->inst_p1 = mono_mempool_alloc (cfg->mempool, sizeof (ArgInfo));
2407 memcpy (ins->inst_p1, ainfo, sizeof (ArgInfo));
2408 MONO_ADD_INS (cfg->cbb, ins);
2412 g_assert_not_reached ();
2417 /* Handle the case where there are no implicit arguments */
2418 if (!sig->pinvoke && (sig->call_convention == MONO_CALL_VARARG) && (cinfo->nargs == sig->sentinelpos))
2419 emit_sig_cookie (cfg, call, cinfo);
2421 call->call_info = cinfo;
2422 call->stack_usage = cinfo->stack_usage;
2426 mono_arch_emit_outarg_vt (MonoCompile *cfg, MonoInst *ins, MonoInst *src)
2428 MonoCallInst *call = (MonoCallInst*)ins->inst_p0;
2429 ArgInfo *ainfo = ins->inst_p1;
2433 if (ins->backend.size == 0 && !ainfo->gsharedvt)
2436 switch (ainfo->storage) {
2437 case ArgVtypeInIRegs:
2438 for (i = 0; i < ainfo->nregs; ++i) {
2439 // FIXME: Smaller sizes
2440 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2441 load->dreg = mono_alloc_ireg (cfg);
2442 load->inst_basereg = src->dreg;
2443 load->inst_offset = i * sizeof(mgreg_t);
2444 MONO_ADD_INS (cfg->cbb, load);
2445 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg + i, load);
2449 for (i = 0; i < ainfo->nregs; ++i) {
2450 if (ainfo->esize == 4)
2451 MONO_INST_NEW (cfg, load, OP_LOADR4_MEMBASE);
2453 MONO_INST_NEW (cfg, load, OP_LOADR8_MEMBASE);
2454 load->dreg = mono_alloc_freg (cfg);
2455 load->inst_basereg = src->dreg;
2456 load->inst_offset = ainfo->foffsets [i];
2457 MONO_ADD_INS (cfg->cbb, load);
2458 add_outarg_reg (cfg, call, ainfo->esize == 4 ? ArgInFRegR4 : ArgInFReg, ainfo->reg + i, load);
2462 case ArgVtypeByRefOnStack: {
2463 MonoInst *vtaddr, *load, *arg;
2465 /* Pass the vtype address in a reg/on the stack */
2466 if (ainfo->gsharedvt) {
2469 /* Make a copy of the argument */
2470 vtaddr = mono_compile_create_var (cfg, &ins->klass->byval_arg, OP_LOCAL);
2472 MONO_INST_NEW (cfg, load, OP_LDADDR);
2473 load->inst_p0 = vtaddr;
2474 vtaddr->flags |= MONO_INST_INDIRECT;
2475 load->type = STACK_MP;
2476 load->klass = vtaddr->klass;
2477 load->dreg = mono_alloc_ireg (cfg);
2478 MONO_ADD_INS (cfg->cbb, load);
2479 mini_emit_memcpy (cfg, load->dreg, 0, src->dreg, 0, ainfo->size, 8);
2482 if (ainfo->storage == ArgVtypeByRef) {
2483 MONO_INST_NEW (cfg, arg, OP_MOVE);
2484 arg->dreg = mono_alloc_preg (cfg);
2485 arg->sreg1 = load->dreg;
2486 MONO_ADD_INS (cfg->cbb, arg);
2487 add_outarg_reg (cfg, call, ArgInIReg, ainfo->reg, arg);
2489 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STORE_MEMBASE_REG, ARMREG_SP, ainfo->offset, load->dreg);
2493 case ArgVtypeOnStack:
2494 for (i = 0; i < ainfo->size / 8; ++i) {
2495 MONO_INST_NEW (cfg, load, OP_LOADI8_MEMBASE);
2496 load->dreg = mono_alloc_ireg (cfg);
2497 load->inst_basereg = src->dreg;
2498 load->inst_offset = i * 8;
2499 MONO_ADD_INS (cfg->cbb, load);
2500 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI8_MEMBASE_REG, ARMREG_SP, ainfo->offset + (i * 8), load->dreg);
2504 g_assert_not_reached ();
2510 mono_arch_emit_setret (MonoCompile *cfg, MonoMethod *method, MonoInst *val)
2512 MonoMethodSignature *sig;
2515 sig = mono_method_signature (cfg->method);
2516 if (!cfg->arch.cinfo)
2517 cfg->arch.cinfo = get_call_info (cfg->mempool, sig);
2518 cinfo = cfg->arch.cinfo;
2520 switch (cinfo->ret.storage) {
2524 MONO_EMIT_NEW_UNALU (cfg, OP_MOVE, cfg->ret->dreg, val->dreg);
2527 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2530 if (COMPILE_LLVM (cfg))
2531 MONO_EMIT_NEW_UNALU (cfg, OP_FMOVE, cfg->ret->dreg, val->dreg);
2533 MONO_EMIT_NEW_UNALU (cfg, OP_RMOVE, cfg->ret->dreg, val->dreg);
2535 MONO_EMIT_NEW_UNALU (cfg, OP_ARM_SETFREG_R4, cfg->ret->dreg, val->dreg);
2538 g_assert_not_reached ();
2544 mono_arch_tail_call_supported (MonoCompile *cfg, MonoMethodSignature *caller_sig, MonoMethodSignature *callee_sig)
2549 if (cfg->compile_aot && !cfg->full_aot)
2550 /* OP_TAILCALL doesn't work with AOT */
2553 c1 = get_call_info (NULL, caller_sig);
2554 c2 = get_call_info (NULL, callee_sig);
2556 // FIXME: Relax these restrictions
2557 if (c1->stack_usage != 0)
2559 if (c1->stack_usage != c2->stack_usage)
2561 if ((c1->ret.storage != ArgNone && c1->ret.storage != ArgInIReg) || c1->ret.storage != c2->ret.storage)
2571 mono_arch_is_inst_imm (gint64 imm)
2573 return (imm >= -((gint64)1<<31) && imm <= (((gint64)1<<31)-1));
2577 mono_arch_instrument_prolog (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments)
2584 mono_arch_instrument_epilog_full (MonoCompile *cfg, void *func, void *p, gboolean enable_arguments, gboolean preserve_argument_registers)
2591 mono_arch_peephole_pass_1 (MonoCompile *cfg, MonoBasicBlock *bb)
2597 mono_arch_peephole_pass_2 (MonoCompile *cfg, MonoBasicBlock *bb)
2602 #define ADD_NEW_INS(cfg,dest,op) do { \
2603 MONO_INST_NEW ((cfg), (dest), (op)); \
2604 mono_bblock_insert_before_ins (bb, ins, (dest)); \
2608 mono_arch_lowering_pass (MonoCompile *cfg, MonoBasicBlock *bb)
2610 MonoInst *ins, *temp, *last_ins = NULL;
2612 MONO_BB_FOR_EACH_INS (bb, ins) {
2613 switch (ins->opcode) {
2618 if (ins->next && (ins->next->opcode == OP_COND_EXC_C || ins->next->opcode == OP_COND_EXC_IC))
2619 /* ARM sets the C flag to 1 if there was _no_ overflow */
2620 ins->next->opcode = OP_COND_EXC_NC;
2624 case OP_IDIV_UN_IMM:
2625 case OP_IREM_UN_IMM:
2627 mono_decompose_op_imm (cfg, bb, ins);
2629 case OP_LOCALLOC_IMM:
2630 if (ins->inst_imm > 32) {
2631 ADD_NEW_INS (cfg, temp, OP_ICONST);
2632 temp->inst_c0 = ins->inst_imm;
2633 temp->dreg = mono_alloc_ireg (cfg);
2634 ins->sreg1 = temp->dreg;
2635 ins->opcode = mono_op_imm_to_op (ins->opcode);
2638 case OP_ICOMPARE_IMM:
2639 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBEQ) {
2640 ins->next->opcode = OP_ARM64_CBZW;
2641 ins->next->sreg1 = ins->sreg1;
2643 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_IBNE_UN) {
2644 ins->next->opcode = OP_ARM64_CBNZW;
2645 ins->next->sreg1 = ins->sreg1;
2649 case OP_LCOMPARE_IMM:
2650 case OP_COMPARE_IMM:
2651 if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBEQ) {
2652 ins->next->opcode = OP_ARM64_CBZX;
2653 ins->next->sreg1 = ins->sreg1;
2655 } else if (ins->inst_imm == 0 && ins->next && ins->next->opcode == OP_LBNE_UN) {
2656 ins->next->opcode = OP_ARM64_CBNZX;
2657 ins->next->sreg1 = ins->sreg1;
2662 gboolean swap = FALSE;
2666 /* Optimized away */
2672 * FP compares with unordered operands set the flags
2673 * to NZCV=0011, which matches some non-unordered compares
2674 * as well, like LE, so have to swap the operands.
2676 switch (ins->next->opcode) {
2678 ins->next->opcode = OP_FBGT;
2682 ins->next->opcode = OP_FBGE;
2690 ins->sreg1 = ins->sreg2;
2701 bb->last_ins = last_ins;
2702 bb->max_vreg = cfg->next_vreg;
2706 mono_arch_decompose_long_opts (MonoCompile *cfg, MonoInst *long_ins)
2711 opcode_to_armcond (int opcode)
2722 case OP_COND_EXC_IEQ:
2723 case OP_COND_EXC_EQ:
2740 case OP_COND_EXC_IGT:
2741 case OP_COND_EXC_GT:
2756 case OP_COND_EXC_ILT:
2757 case OP_COND_EXC_LT:
2765 case OP_COND_EXC_INE_UN:
2766 case OP_COND_EXC_NE_UN:
2772 case OP_COND_EXC_IGE_UN:
2773 case OP_COND_EXC_GE_UN:
2783 case OP_COND_EXC_IGT_UN:
2784 case OP_COND_EXC_GT_UN:
2790 case OP_COND_EXC_ILE_UN:
2791 case OP_COND_EXC_LE_UN:
2799 case OP_COND_EXC_ILT_UN:
2800 case OP_COND_EXC_LT_UN:
2803 * FCMP sets the NZCV condition bits as follows:
2808 * ARMCOND_LT is N!=V, so it matches unordered too, so
2809 * fclt and fclt_un need to be special cased.
2819 case OP_COND_EXC_IC:
2821 case OP_COND_EXC_OV:
2822 case OP_COND_EXC_IOV:
2824 case OP_COND_EXC_NC:
2825 case OP_COND_EXC_INC:
2827 case OP_COND_EXC_NO:
2828 case OP_COND_EXC_INO:
2831 printf ("%s\n", mono_inst_name (opcode));
2832 g_assert_not_reached ();
2837 /* This clobbers LR */
2838 static inline __attribute__ ((__warn_unused_result__)) guint8*
2839 emit_cond_exc (MonoCompile *cfg, guint8 *code, int opcode, const char *exc_name)
2843 cond = opcode_to_armcond (opcode);
2845 arm_adrx (code, ARMREG_IP1, code);
2846 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_EXC, exc_name, MONO_R_ARM64_BCC);
2847 arm_bcc (code, cond, 0);
2852 emit_move_return_value (MonoCompile *cfg, guint8 * code, MonoInst *ins)
2857 call = (MonoCallInst*)ins;
2858 cinfo = call->call_info;
2860 switch (cinfo->ret.storage) {
2864 /* LLVM compiled code might only set the bottom bits */
2865 if (call->signature && mini_get_underlying_type (call->signature->ret)->type == MONO_TYPE_I4)
2866 arm_sxtwx (code, call->inst.dreg, cinfo->ret.reg);
2867 else if (call->inst.dreg != cinfo->ret.reg)
2868 arm_movx (code, call->inst.dreg, cinfo->ret.reg);
2871 if (call->inst.dreg != cinfo->ret.reg)
2872 arm_fmovd (code, call->inst.dreg, cinfo->ret.reg);
2876 arm_fmovs (code, call->inst.dreg, cinfo->ret.reg);
2878 arm_fcvt_sd (code, call->inst.dreg, cinfo->ret.reg);
2880 case ArgVtypeInIRegs: {
2881 MonoInst *loc = cfg->arch.vret_addr_loc;
2884 /* Load the destination address */
2885 g_assert (loc && loc->opcode == OP_REGOFFSET);
2886 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
2887 for (i = 0; i < cinfo->ret.nregs; ++i)
2888 arm_strx (code, cinfo->ret.reg + i, ARMREG_LR, i * 8);
2892 MonoInst *loc = cfg->arch.vret_addr_loc;
2895 /* Load the destination address */
2896 g_assert (loc && loc->opcode == OP_REGOFFSET);
2897 code = emit_ldrx (code, ARMREG_LR, loc->inst_basereg, loc->inst_offset);
2898 for (i = 0; i < cinfo->ret.nregs; ++i) {
2899 if (cinfo->ret.esize == 4)
2900 arm_strfpw (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
2902 arm_strfpx (code, cinfo->ret.reg + i, ARMREG_LR, cinfo->ret.foffsets [i]);
2909 g_assert_not_reached ();
2916 * emit_branch_island:
2918 * Emit a branch island for the conditional branches from cfg->native_code + start_offset to code.
2921 emit_branch_island (MonoCompile *cfg, guint8 *code, int start_offset)
2924 int offset, island_size;
2926 /* Iterate over the patch infos added so far by this bb */
2928 for (ji = cfg->patch_info; ji; ji = ji->next) {
2929 if (ji->ip.i < start_offset)
2930 /* The patch infos are in reverse order, so this means the end */
2932 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ)
2937 offset = code - cfg->native_code;
2938 if (offset > (cfg->code_size - island_size - 16)) {
2939 cfg->code_size *= 2;
2940 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
2941 code = cfg->native_code + offset;
2944 /* Branch over the island */
2945 arm_b (code, code + 4 + island_size);
2947 for (ji = cfg->patch_info; ji; ji = ji->next) {
2948 if (ji->ip.i < start_offset)
2950 if (ji->relocation == MONO_R_ARM64_BCC || ji->relocation == MONO_R_ARM64_CBZ) {
2951 /* Rewrite the cond branch so it branches to an uncoditional branch in the branch island */
2952 arm_patch_rel (cfg->native_code + ji->ip.i, code, ji->relocation);
2953 /* Rewrite the patch so it points to the unconditional branch */
2954 ji->ip.i = code - cfg->native_code;
2955 ji->relocation = MONO_R_ARM64_B;
2964 mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb)
2969 guint8 *code = cfg->native_code + cfg->code_len;
2970 int start_offset, max_len, dreg, sreg1, sreg2;
2973 if (cfg->verbose_level > 2)
2974 g_print ("Basic block %d starting at offset 0x%x\n", bb->block_num, bb->native_offset);
2976 start_offset = code - cfg->native_code;
2978 MONO_BB_FOR_EACH_INS (bb, ins) {
2979 offset = code - cfg->native_code;
2981 max_len = ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
2983 if (offset > (cfg->code_size - max_len - 16)) {
2984 cfg->code_size *= 2;
2985 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
2986 code = cfg->native_code + offset;
2989 if (G_UNLIKELY (cfg->arch.cond_branch_islands && offset - start_offset > 4 * 0x1ffff)) {
2990 /* Emit a branch island for large basic blocks */
2991 code = emit_branch_island (cfg, code, start_offset);
2992 offset = code - cfg->native_code;
2993 start_offset = offset;
2996 mono_debug_record_line_number (cfg, ins, offset);
3001 imm = ins->inst_imm;
3003 switch (ins->opcode) {
3005 code = emit_imm (code, dreg, ins->inst_c0);
3008 code = emit_imm64 (code, dreg, ins->inst_c0);
3012 arm_movx (code, dreg, sreg1);
3015 case OP_RELAXED_NOP:
3018 mono_add_patch_info_rel (cfg, offset, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0, MONO_R_ARM64_IMM);
3019 code = emit_imm64_template (code, dreg);
3023 * gdb does not like encountering the hw breakpoint ins in the debugged code.
3024 * So instead of emitting a trap, we emit a call a C function and place a
3027 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, (gpointer)"mono_break");
3032 arm_addx_imm (code, ARMREG_IP0, sreg1, (MONO_ARCH_FRAME_ALIGNMENT - 1));
3033 // FIXME: andx_imm doesn't work yet
3034 code = emit_imm (code, ARMREG_IP1, -MONO_ARCH_FRAME_ALIGNMENT);
3035 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3036 //arm_andx_imm (code, ARMREG_IP0, sreg1, - MONO_ARCH_FRAME_ALIGNMENT);
3037 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
3038 arm_subx (code, ARMREG_IP1, ARMREG_IP1, ARMREG_IP0);
3039 arm_movspx (code, ARMREG_SP, ARMREG_IP1);
3042 /* ip1 = pointer, ip0 = end */
3043 arm_addx (code, ARMREG_IP0, ARMREG_IP1, ARMREG_IP0);
3045 arm_cmpx (code, ARMREG_IP1, ARMREG_IP0);
3047 arm_bcc (code, ARMCOND_EQ, 0);
3048 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_IP1, 0);
3049 arm_addx_imm (code, ARMREG_IP1, ARMREG_IP1, 16);
3050 arm_b (code, buf [0]);
3051 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3053 arm_movspx (code, dreg, ARMREG_SP);
3054 if (cfg->param_area)
3055 code = emit_subx_sp_imm (code, cfg->param_area);
3058 case OP_LOCALLOC_IMM: {
3061 imm = ALIGN_TO (ins->inst_imm, MONO_ARCH_FRAME_ALIGNMENT);
3062 g_assert (arm_is_arith_imm (imm));
3063 arm_subx_imm (code, ARMREG_SP, ARMREG_SP, imm);
3066 g_assert (MONO_ARCH_FRAME_ALIGNMENT == 16);
3068 while (offset < imm) {
3069 arm_stpx (code, ARMREG_RZR, ARMREG_RZR, ARMREG_SP, offset);
3072 arm_movspx (code, dreg, ARMREG_SP);
3073 if (cfg->param_area)
3074 code = emit_subx_sp_imm (code, cfg->param_area);
3078 code = emit_aotconst (cfg, code, dreg, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3080 case OP_OBJC_GET_SELECTOR:
3081 mono_add_patch_info (cfg, offset, MONO_PATCH_INFO_OBJC_SELECTOR_REF, ins->inst_p0);
3082 /* See arch_emit_objc_selector_ref () in aot-compiler.c */
3083 arm_ldrx_lit (code, ins->dreg, 0);
3087 case OP_SEQ_POINT: {
3088 MonoInst *info_var = cfg->arch.seq_point_info_var;
3091 * For AOT, we use one got slot per method, which will point to a
3092 * SeqPointInfo structure, containing all the information required
3093 * by the code below.
3095 if (cfg->compile_aot) {
3096 g_assert (info_var);
3097 g_assert (info_var->opcode == OP_REGOFFSET);
3100 if (ins->flags & MONO_INST_SINGLE_STEP_LOC) {
3101 MonoInst *var = cfg->arch.ss_tramp_var;
3104 g_assert (var->opcode == OP_REGOFFSET);
3105 /* Load ss_tramp_var */
3106 /* This is equal to &ss_trampoline */
3107 arm_ldrx (code, ARMREG_IP1, var->inst_basereg, var->inst_offset);
3108 /* Load the trampoline address */
3109 arm_ldrx (code, ARMREG_IP1, ARMREG_IP1, 0);
3110 /* Call it if it is non-null */
3111 arm_cbzx (code, ARMREG_IP1, code + 8);
3112 arm_blrx (code, ARMREG_IP1);
3115 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
3117 if (cfg->compile_aot) {
3118 guint32 offset = code - cfg->native_code;
3121 arm_ldrx (code, ARMREG_IP1, info_var->inst_basereg, info_var->inst_offset);
3122 /* Add the offset */
3123 val = ((offset / 4) * sizeof (guint8*)) + MONO_STRUCT_OFFSET (SeqPointInfo, bp_addrs);
3124 /* Load the info->bp_addrs [offset], which is either 0 or the address of the bp trampoline */
3125 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP1, val);
3126 /* Skip the load if its 0 */
3127 arm_cbzx (code, ARMREG_IP1, code + 8);
3128 /* Call the breakpoint trampoline */
3129 arm_blrx (code, ARMREG_IP1);
3131 MonoInst *var = cfg->arch.bp_tramp_var;
3134 g_assert (var->opcode == OP_REGOFFSET);
3135 /* Load the address of the bp trampoline into IP0 */
3136 arm_ldrx (code, ARMREG_IP0, var->inst_basereg, var->inst_offset);
3138 * A placeholder for a possible breakpoint inserted by
3139 * mono_arch_set_breakpoint ().
3148 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_B);
3152 arm_brx (code, sreg1);
3184 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3185 cond = opcode_to_armcond (ins->opcode);
3186 arm_bcc (code, cond, 0);
3190 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3191 /* For fp compares, ARMCOND_LT is lt or unordered */
3192 arm_bcc (code, ARMCOND_LT, 0);
3195 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3196 arm_bcc (code, ARMCOND_EQ, 0);
3197 offset = code - cfg->native_code;
3198 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_BCC);
3199 /* For fp compares, ARMCOND_LT is lt or unordered */
3200 arm_bcc (code, ARMCOND_LT, 0);
3203 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3204 arm_cbzw (code, sreg1, 0);
3207 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3208 arm_cbzx (code, sreg1, 0);
3210 case OP_ARM64_CBNZW:
3211 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3212 arm_cbnzw (code, sreg1, 0);
3214 case OP_ARM64_CBNZX:
3215 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_true_bb, MONO_R_ARM64_CBZ);
3216 arm_cbnzx (code, sreg1, 0);
3220 arm_addw (code, dreg, sreg1, sreg2);
3223 arm_addx (code, dreg, sreg1, sreg2);
3226 arm_subw (code, dreg, sreg1, sreg2);
3229 arm_subx (code, dreg, sreg1, sreg2);
3232 arm_andw (code, dreg, sreg1, sreg2);
3235 arm_andx (code, dreg, sreg1, sreg2);
3238 arm_orrw (code, dreg, sreg1, sreg2);
3241 arm_orrx (code, dreg, sreg1, sreg2);
3244 arm_eorw (code, dreg, sreg1, sreg2);
3247 arm_eorx (code, dreg, sreg1, sreg2);
3250 arm_negw (code, dreg, sreg1);
3253 arm_negx (code, dreg, sreg1);
3256 arm_mvnw (code, dreg, sreg1);
3259 arm_mvnx (code, dreg, sreg1);
3262 arm_addsw (code, dreg, sreg1, sreg2);
3266 arm_addsx (code, dreg, sreg1, sreg2);
3269 arm_subsw (code, dreg, sreg1, sreg2);
3273 arm_subsx (code, dreg, sreg1, sreg2);
3276 arm_cmpw (code, sreg1, sreg2);
3280 arm_cmpx (code, sreg1, sreg2);
3283 code = emit_addw_imm (code, dreg, sreg1, imm);
3287 code = emit_addx_imm (code, dreg, sreg1, imm);
3290 code = emit_subw_imm (code, dreg, sreg1, imm);
3293 code = emit_subx_imm (code, dreg, sreg1, imm);
3296 code = emit_andw_imm (code, dreg, sreg1, imm);
3300 code = emit_andx_imm (code, dreg, sreg1, imm);
3303 code = emit_orrw_imm (code, dreg, sreg1, imm);
3306 code = emit_orrx_imm (code, dreg, sreg1, imm);
3309 code = emit_eorw_imm (code, dreg, sreg1, imm);
3312 code = emit_eorx_imm (code, dreg, sreg1, imm);
3314 case OP_ICOMPARE_IMM:
3315 code = emit_cmpw_imm (code, sreg1, imm);
3317 case OP_LCOMPARE_IMM:
3318 case OP_COMPARE_IMM:
3320 arm_cmpx (code, sreg1, ARMREG_RZR);
3322 // FIXME: 32 vs 64 bit issues for 0xffffffff
3323 code = emit_imm64 (code, ARMREG_LR, imm);
3324 arm_cmpx (code, sreg1, ARMREG_LR);
3328 arm_lslvw (code, dreg, sreg1, sreg2);
3331 arm_lslvx (code, dreg, sreg1, sreg2);
3334 arm_asrvw (code, dreg, sreg1, sreg2);
3337 arm_asrvx (code, dreg, sreg1, sreg2);
3340 arm_lsrvw (code, dreg, sreg1, sreg2);
3343 arm_lsrvx (code, dreg, sreg1, sreg2);
3347 arm_movx (code, dreg, sreg1);
3349 arm_lslw (code, dreg, sreg1, imm);
3353 arm_movx (code, dreg, sreg1);
3355 arm_lslx (code, dreg, sreg1, imm);
3359 arm_movx (code, dreg, sreg1);
3361 arm_asrw (code, dreg, sreg1, imm);
3366 arm_movx (code, dreg, sreg1);
3368 arm_asrx (code, dreg, sreg1, imm);
3370 case OP_ISHR_UN_IMM:
3372 arm_movx (code, dreg, sreg1);
3374 arm_lsrw (code, dreg, sreg1, imm);
3377 case OP_LSHR_UN_IMM:
3379 arm_movx (code, dreg, sreg1);
3381 arm_lsrx (code, dreg, sreg1, imm);
3386 arm_sxtwx (code, dreg, sreg1);
3389 /* Clean out the upper word */
3390 arm_movw (code, dreg, sreg1);
3393 arm_lslx (code, dreg, sreg1, imm);
3396 /* MULTIPLY/DIVISION */
3399 // FIXME: Optimize this
3400 /* Check for zero */
3401 arm_cmpx_imm (code, sreg2, 0);
3402 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3403 /* Check for INT_MIN/-1 */
3404 code = emit_imm (code, ARMREG_IP0, 0x80000000);
3405 arm_cmpx (code, sreg1, ARMREG_IP0);
3406 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3407 code = emit_imm (code, ARMREG_IP0, 0xffffffff);
3408 arm_cmpx (code, sreg2, ARMREG_IP0);
3409 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3410 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3411 arm_cmpx_imm (code, ARMREG_IP0, 1);
3412 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "OverflowException");
3413 if (ins->opcode == OP_IREM) {
3414 arm_sdivw (code, ARMREG_LR, sreg1, sreg2);
3415 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3417 arm_sdivw (code, dreg, sreg1, sreg2);
3421 arm_cmpx_imm (code, sreg2, 0);
3422 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3423 arm_udivw (code, dreg, sreg1, sreg2);
3426 arm_cmpx_imm (code, sreg2, 0);
3427 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3428 arm_udivw (code, ARMREG_LR, sreg1, sreg2);
3429 arm_msubw (code, dreg, ARMREG_LR, sreg2, sreg1);
3433 // FIXME: Optimize this
3434 /* Check for zero */
3435 arm_cmpx_imm (code, sreg2, 0);
3436 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3437 /* Check for INT64_MIN/-1 */
3438 code = emit_imm64 (code, ARMREG_IP0, 0x8000000000000000);
3439 arm_cmpx (code, sreg1, ARMREG_IP0);
3440 arm_cset (code, ARMCOND_EQ, ARMREG_IP1);
3441 code = emit_imm64 (code, ARMREG_IP0, 0xffffffffffffffff);
3442 arm_cmpx (code, sreg2, ARMREG_IP0);
3443 arm_cset (code, ARMCOND_EQ, ARMREG_IP0);
3444 arm_andx (code, ARMREG_IP0, ARMREG_IP0, ARMREG_IP1);
3445 arm_cmpx_imm (code, ARMREG_IP0, 1);
3446 /* 64 bit uses ArithmeticException */
3447 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "ArithmeticException");
3448 if (ins->opcode == OP_LREM) {
3449 arm_sdivx (code, ARMREG_LR, sreg1, sreg2);
3450 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3452 arm_sdivx (code, dreg, sreg1, sreg2);
3456 arm_cmpx_imm (code, sreg2, 0);
3457 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3458 arm_udivx (code, dreg, sreg1, sreg2);
3461 arm_cmpx_imm (code, sreg2, 0);
3462 code = emit_cond_exc (cfg, code, OP_COND_EXC_IEQ, "DivideByZeroException");
3463 arm_udivx (code, ARMREG_LR, sreg1, sreg2);
3464 arm_msubx (code, dreg, ARMREG_LR, sreg2, sreg1);
3467 arm_mulw (code, dreg, sreg1, sreg2);
3470 arm_mulx (code, dreg, sreg1, sreg2);
3473 code = emit_imm (code, ARMREG_LR, imm);
3474 arm_mulw (code, dreg, sreg1, ARMREG_LR);
3478 code = emit_imm (code, ARMREG_LR, imm);
3479 arm_mulx (code, dreg, sreg1, ARMREG_LR);
3483 case OP_ICONV_TO_I1:
3484 case OP_LCONV_TO_I1:
3485 arm_sxtbx (code, dreg, sreg1);
3487 case OP_ICONV_TO_I2:
3488 case OP_LCONV_TO_I2:
3489 arm_sxthx (code, dreg, sreg1);
3491 case OP_ICONV_TO_U1:
3492 case OP_LCONV_TO_U1:
3493 arm_uxtbw (code, dreg, sreg1);
3495 case OP_ICONV_TO_U2:
3496 case OP_LCONV_TO_U2:
3497 arm_uxthw (code, dreg, sreg1);
3523 cond = opcode_to_armcond (ins->opcode);
3524 arm_cset (code, cond, dreg);
3537 cond = opcode_to_armcond (ins->opcode);
3538 arm_fcmpd (code, sreg1, sreg2);
3539 arm_cset (code, cond, dreg);
3544 case OP_LOADI1_MEMBASE:
3545 code = emit_ldrsbx (code, dreg, ins->inst_basereg, ins->inst_offset);
3547 case OP_LOADU1_MEMBASE:
3548 code = emit_ldrb (code, dreg, ins->inst_basereg, ins->inst_offset);
3550 case OP_LOADI2_MEMBASE:
3551 code = emit_ldrshx (code, dreg, ins->inst_basereg, ins->inst_offset);
3553 case OP_LOADU2_MEMBASE:
3554 code = emit_ldrh (code, dreg, ins->inst_basereg, ins->inst_offset);
3556 case OP_LOADI4_MEMBASE:
3557 code = emit_ldrswx (code, dreg, ins->inst_basereg, ins->inst_offset);
3559 case OP_LOADU4_MEMBASE:
3560 code = emit_ldrw (code, dreg, ins->inst_basereg, ins->inst_offset);
3562 case OP_LOAD_MEMBASE:
3563 case OP_LOADI8_MEMBASE:
3564 code = emit_ldrx (code, dreg, ins->inst_basereg, ins->inst_offset);
3566 case OP_STOREI1_MEMBASE_IMM:
3567 case OP_STOREI2_MEMBASE_IMM:
3568 case OP_STOREI4_MEMBASE_IMM:
3569 case OP_STORE_MEMBASE_IMM:
3570 case OP_STOREI8_MEMBASE_IMM: {
3574 code = emit_imm (code, ARMREG_LR, imm);
3577 immreg = ARMREG_RZR;
3580 switch (ins->opcode) {
3581 case OP_STOREI1_MEMBASE_IMM:
3582 code = emit_strb (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3584 case OP_STOREI2_MEMBASE_IMM:
3585 code = emit_strh (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3587 case OP_STOREI4_MEMBASE_IMM:
3588 code = emit_strw (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3590 case OP_STORE_MEMBASE_IMM:
3591 case OP_STOREI8_MEMBASE_IMM:
3592 code = emit_strx (code, immreg, ins->inst_destbasereg, ins->inst_offset);
3595 g_assert_not_reached ();
3600 case OP_STOREI1_MEMBASE_REG:
3601 code = emit_strb (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3603 case OP_STOREI2_MEMBASE_REG:
3604 code = emit_strh (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3606 case OP_STOREI4_MEMBASE_REG:
3607 code = emit_strw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3609 case OP_STORE_MEMBASE_REG:
3610 case OP_STOREI8_MEMBASE_REG:
3611 code = emit_strx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3614 code = emit_tls_get (code, dreg, ins->inst_offset);
3617 code = emit_tls_set (code, sreg1, ins->inst_offset);
3620 case OP_MEMORY_BARRIER:
3623 case OP_ATOMIC_ADD_I4: {
3627 arm_ldxrw (code, ARMREG_IP0, sreg1);
3628 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3629 arm_stlxrw (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3630 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3633 arm_movx (code, dreg, ARMREG_IP0);
3636 case OP_ATOMIC_ADD_I8: {
3640 arm_ldxrx (code, ARMREG_IP0, sreg1);
3641 arm_addx (code, ARMREG_IP0, ARMREG_IP0, sreg2);
3642 arm_stlxrx (code, ARMREG_IP1, ARMREG_IP0, sreg1);
3643 arm_cbnzx (code, ARMREG_IP1, buf [0]);
3646 arm_movx (code, dreg, ARMREG_IP0);
3649 case OP_ATOMIC_EXCHANGE_I4: {
3653 arm_ldxrw (code, ARMREG_IP0, sreg1);
3654 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3655 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3658 arm_movx (code, dreg, ARMREG_IP0);
3661 case OP_ATOMIC_EXCHANGE_I8: {
3665 arm_ldxrx (code, ARMREG_IP0, sreg1);
3666 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3667 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3670 arm_movx (code, dreg, ARMREG_IP0);
3673 case OP_ATOMIC_CAS_I4: {
3676 /* sreg2 is the value, sreg3 is the comparand */
3678 arm_ldxrw (code, ARMREG_IP0, sreg1);
3679 arm_cmpw (code, ARMREG_IP0, ins->sreg3);
3681 arm_bcc (code, ARMCOND_NE, 0);
3682 arm_stlxrw (code, ARMREG_IP1, sreg2, sreg1);
3683 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3684 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3687 arm_movx (code, dreg, ARMREG_IP0);
3690 case OP_ATOMIC_CAS_I8: {
3694 arm_ldxrx (code, ARMREG_IP0, sreg1);
3695 arm_cmpx (code, ARMREG_IP0, ins->sreg3);
3697 arm_bcc (code, ARMCOND_NE, 0);
3698 arm_stlxrx (code, ARMREG_IP1, sreg2, sreg1);
3699 arm_cbnzw (code, ARMREG_IP1, buf [0]);
3700 arm_patch_rel (buf [1], code, MONO_R_ARM64_BCC);
3703 arm_movx (code, dreg, ARMREG_IP0);
3706 case OP_ATOMIC_LOAD_I1: {
3707 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3708 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3710 arm_ldarb (code, ins->dreg, ARMREG_LR);
3711 arm_sxtbx (code, ins->dreg, ins->dreg);
3714 case OP_ATOMIC_LOAD_U1: {
3715 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3716 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3718 arm_ldarb (code, ins->dreg, ARMREG_LR);
3719 arm_uxtbx (code, ins->dreg, ins->dreg);
3722 case OP_ATOMIC_LOAD_I2: {
3723 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3724 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3726 arm_ldarh (code, ins->dreg, ARMREG_LR);
3727 arm_sxthx (code, ins->dreg, ins->dreg);
3730 case OP_ATOMIC_LOAD_U2: {
3731 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3732 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3734 arm_ldarh (code, ins->dreg, ARMREG_LR);
3735 arm_uxthx (code, ins->dreg, ins->dreg);
3738 case OP_ATOMIC_LOAD_I4: {
3739 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3740 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3742 arm_ldarw (code, ins->dreg, ARMREG_LR);
3743 arm_sxtwx (code, ins->dreg, ins->dreg);
3746 case OP_ATOMIC_LOAD_U4: {
3747 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3748 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3750 arm_ldarw (code, ins->dreg, ARMREG_LR);
3751 arm_movw (code, ins->dreg, ins->dreg); /* Clear upper half of the register. */
3754 case OP_ATOMIC_LOAD_I8:
3755 case OP_ATOMIC_LOAD_U8: {
3756 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3757 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3759 arm_ldarx (code, ins->dreg, ARMREG_LR);
3762 case OP_ATOMIC_LOAD_R4: {
3763 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3764 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3767 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3768 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3770 arm_ldarw (code, ARMREG_LR, ARMREG_LR);
3771 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3772 arm_fcvt_sd (code, ins->dreg, FP_TEMP_REG);
3776 case OP_ATOMIC_LOAD_R8: {
3777 code = emit_addx_imm (code, ARMREG_LR, ins->inst_basereg, ins->inst_offset);
3778 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3780 arm_ldarx (code, ARMREG_LR, ARMREG_LR);
3781 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3784 case OP_ATOMIC_STORE_I1:
3785 case OP_ATOMIC_STORE_U1: {
3786 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3787 arm_stlrb (code, ARMREG_LR, ins->sreg1);
3788 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3792 case OP_ATOMIC_STORE_I2:
3793 case OP_ATOMIC_STORE_U2: {
3794 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3795 arm_stlrh (code, ARMREG_LR, ins->sreg1);
3796 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3800 case OP_ATOMIC_STORE_I4:
3801 case OP_ATOMIC_STORE_U4: {
3802 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3803 arm_stlrw (code, ARMREG_LR, ins->sreg1);
3804 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3808 case OP_ATOMIC_STORE_I8:
3809 case OP_ATOMIC_STORE_U8: {
3810 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3811 arm_stlrx (code, ARMREG_LR, ins->sreg1);
3812 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3816 case OP_ATOMIC_STORE_R4: {
3817 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3819 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
3820 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
3822 arm_fcvt_ds (code, FP_TEMP_REG, ins->sreg1);
3823 arm_fmov_double_to_rx (code, ARMREG_IP0, FP_TEMP_REG);
3824 arm_stlrw (code, ARMREG_LR, ARMREG_IP0);
3826 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3830 case OP_ATOMIC_STORE_R8: {
3831 code = emit_addx_imm (code, ARMREG_LR, ins->inst_destbasereg, ins->inst_offset);
3832 arm_fmov_double_to_rx (code, ARMREG_IP0, ins->sreg1);
3833 arm_stlrx (code, ARMREG_LR, ARMREG_IP0);
3834 if (ins->backend.memory_barrier_kind == MONO_MEMORY_BARRIER_SEQ)
3841 guint64 imm = *(guint64*)ins->inst_p0;
3844 arm_fmov_rx_to_double (code, dreg, ARMREG_RZR);
3846 code = emit_imm64 (code, ARMREG_LR, imm);
3847 arm_fmov_rx_to_double (code, ins->dreg, ARMREG_LR);
3852 guint64 imm = *(guint32*)ins->inst_p0;
3854 code = emit_imm64 (code, ARMREG_LR, imm);
3856 arm_fmov_rx_to_double (code, dreg, ARMREG_LR);
3858 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
3859 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3863 case OP_LOADR8_MEMBASE:
3864 code = emit_ldrfpx (code, dreg, ins->inst_basereg, ins->inst_offset);
3866 case OP_LOADR4_MEMBASE:
3868 code = emit_ldrfpw (code, dreg, ins->inst_basereg, ins->inst_offset);
3870 code = emit_ldrfpw (code, FP_TEMP_REG, ins->inst_basereg, ins->inst_offset);
3871 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3874 case OP_STORER8_MEMBASE_REG:
3875 code = emit_strfpx (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3877 case OP_STORER4_MEMBASE_REG:
3879 code = emit_strfpw (code, sreg1, ins->inst_destbasereg, ins->inst_offset);
3881 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
3882 code = emit_strfpw (code, FP_TEMP_REG, ins->inst_destbasereg, ins->inst_offset);
3887 arm_fmovd (code, dreg, sreg1);
3891 arm_fmovs (code, dreg, sreg1);
3893 case OP_MOVE_F_TO_I4:
3895 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
3897 arm_fcvt_ds (code, ins->dreg, ins->sreg1);
3898 arm_fmov_double_to_rx (code, ins->dreg, ins->dreg);
3901 case OP_MOVE_I4_TO_F:
3903 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3905 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3906 arm_fcvt_sd (code, ins->dreg, ins->dreg);
3909 case OP_MOVE_F_TO_I8:
3910 arm_fmov_double_to_rx (code, ins->dreg, ins->sreg1);
3912 case OP_MOVE_I8_TO_F:
3913 arm_fmov_rx_to_double (code, ins->dreg, ins->sreg1);
3916 arm_fcmpd (code, sreg1, sreg2);
3919 arm_fcmps (code, sreg1, sreg2);
3921 case OP_FCONV_TO_I1:
3922 arm_fcvtzs_dx (code, dreg, sreg1);
3923 arm_sxtbx (code, dreg, dreg);
3925 case OP_FCONV_TO_U1:
3926 arm_fcvtzu_dx (code, dreg, sreg1);
3927 arm_uxtbw (code, dreg, dreg);
3929 case OP_FCONV_TO_I2:
3930 arm_fcvtzs_dx (code, dreg, sreg1);
3931 arm_sxthx (code, dreg, dreg);
3933 case OP_FCONV_TO_U2:
3934 arm_fcvtzu_dx (code, dreg, sreg1);
3935 arm_uxthw (code, dreg, dreg);
3937 case OP_FCONV_TO_I4:
3938 arm_fcvtzs_dx (code, dreg, sreg1);
3939 arm_sxtwx (code, dreg, dreg);
3941 case OP_FCONV_TO_U4:
3942 arm_fcvtzu_dx (code, dreg, sreg1);
3944 case OP_FCONV_TO_I8:
3945 arm_fcvtzs_dx (code, dreg, sreg1);
3947 case OP_FCONV_TO_U8:
3948 arm_fcvtzu_dx (code, dreg, sreg1);
3950 case OP_FCONV_TO_R4:
3952 arm_fcvt_ds (code, dreg, sreg1);
3954 arm_fcvt_ds (code, FP_TEMP_REG, sreg1);
3955 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3958 case OP_ICONV_TO_R4:
3960 arm_scvtf_rw_to_s (code, dreg, sreg1);
3962 arm_scvtf_rw_to_s (code, FP_TEMP_REG, sreg1);
3963 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3966 case OP_LCONV_TO_R4:
3968 arm_scvtf_rx_to_s (code, dreg, sreg1);
3970 arm_scvtf_rx_to_s (code, FP_TEMP_REG, sreg1);
3971 arm_fcvt_sd (code, dreg, FP_TEMP_REG);
3974 case OP_ICONV_TO_R8:
3975 arm_scvtf_rw_to_d (code, dreg, sreg1);
3977 case OP_LCONV_TO_R8:
3978 arm_scvtf_rx_to_d (code, dreg, sreg1);
3980 case OP_ICONV_TO_R_UN:
3981 arm_ucvtf_rw_to_d (code, dreg, sreg1);
3983 case OP_LCONV_TO_R_UN:
3984 arm_ucvtf_rx_to_d (code, dreg, sreg1);
3987 arm_fadd_d (code, dreg, sreg1, sreg2);
3990 arm_fsub_d (code, dreg, sreg1, sreg2);
3993 arm_fmul_d (code, dreg, sreg1, sreg2);
3996 arm_fdiv_d (code, dreg, sreg1, sreg2);
4000 g_assert_not_reached ();
4003 arm_fneg_d (code, dreg, sreg1);
4005 case OP_ARM_SETFREG_R4:
4006 arm_fcvt_ds (code, dreg, sreg1);
4009 /* Check for infinity */
4010 code = emit_imm64 (code, ARMREG_LR, 0x7fefffffffffffffLL);
4011 arm_fmov_rx_to_double (code, FP_TEMP_REG, ARMREG_LR);
4012 arm_fabs_d (code, FP_TEMP_REG2, sreg1);
4013 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG);
4014 code = emit_cond_exc (cfg, code, OP_COND_EXC_GT, "ArithmeticException");
4015 /* Check for nans */
4016 arm_fcmpd (code, FP_TEMP_REG2, FP_TEMP_REG2);
4017 code = emit_cond_exc (cfg, code, OP_COND_EXC_OV, "ArithmeticException");
4018 arm_fmovd (code, dreg, sreg1);
4023 arm_fadd_s (code, dreg, sreg1, sreg2);
4026 arm_fsub_s (code, dreg, sreg1, sreg2);
4029 arm_fmul_s (code, dreg, sreg1, sreg2);
4032 arm_fdiv_s (code, dreg, sreg1, sreg2);
4035 arm_fneg_s (code, dreg, sreg1);
4037 case OP_RCONV_TO_I1:
4038 arm_fcvtzs_sx (code, dreg, sreg1);
4039 arm_sxtbx (code, dreg, dreg);
4041 case OP_RCONV_TO_U1:
4042 arm_fcvtzu_sx (code, dreg, sreg1);
4043 arm_uxtbw (code, dreg, dreg);
4045 case OP_RCONV_TO_I2:
4046 arm_fcvtzs_sx (code, dreg, sreg1);
4047 arm_sxthx (code, dreg, dreg);
4049 case OP_RCONV_TO_U2:
4050 arm_fcvtzu_sx (code, dreg, sreg1);
4051 arm_uxthw (code, dreg, dreg);
4053 case OP_RCONV_TO_I4:
4054 arm_fcvtzs_sx (code, dreg, sreg1);
4055 arm_sxtwx (code, dreg, dreg);
4057 case OP_RCONV_TO_U4:
4058 arm_fcvtzu_sx (code, dreg, sreg1);
4060 case OP_RCONV_TO_I8:
4061 arm_fcvtzs_sx (code, dreg, sreg1);
4063 case OP_RCONV_TO_U8:
4064 arm_fcvtzu_sx (code, dreg, sreg1);
4066 case OP_RCONV_TO_R8:
4067 arm_fcvt_sd (code, dreg, sreg1);
4069 case OP_RCONV_TO_R4:
4071 arm_fmovs (code, dreg, sreg1);
4083 cond = opcode_to_armcond (ins->opcode);
4084 arm_fcmps (code, sreg1, sreg2);
4085 arm_cset (code, cond, dreg);
4096 call = (MonoCallInst*)ins;
4097 if (ins->flags & MONO_INST_HAS_METHOD)
4098 code = emit_call (cfg, code, MONO_PATCH_INFO_METHOD, call->method);
4100 code = emit_call (cfg, code, MONO_PATCH_INFO_ABS, call->fptr);
4101 code = emit_move_return_value (cfg, code, ins);
4103 case OP_VOIDCALL_REG:
4109 arm_blrx (code, sreg1);
4110 code = emit_move_return_value (cfg, code, ins);
4112 case OP_VOIDCALL_MEMBASE:
4113 case OP_CALL_MEMBASE:
4114 case OP_LCALL_MEMBASE:
4115 case OP_FCALL_MEMBASE:
4116 case OP_RCALL_MEMBASE:
4117 case OP_VCALL2_MEMBASE:
4118 code = emit_ldrx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4119 arm_blrx (code, ARMREG_IP0);
4120 code = emit_move_return_value (cfg, code, ins);
4123 MonoCallInst *call = (MonoCallInst*)ins;
4125 g_assert (!cfg->method->save_lmf);
4127 // FIXME: Copy stack arguments
4129 /* Restore registers */
4130 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4133 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, ((1 << ARMREG_IP0) | (1 << ARMREG_IP1)));
4135 if (cfg->compile_aot) {
4136 /* This is not a PLT patch */
4137 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_METHOD_JUMP, call->method);
4138 arm_brx (code, ARMREG_IP0);
4140 mono_add_patch_info_rel (cfg, code - cfg->native_code, MONO_PATCH_INFO_METHOD_JUMP, call->method, MONO_R_ARM64_B);
4142 cfg->thunk_area += THUNK_SIZE;
4144 ins->flags |= MONO_INST_GC_CALLSITE;
4145 ins->backend.pc_offset = code - cfg->native_code;
4149 g_assert (cfg->arch.cinfo);
4150 code = emit_addx_imm (code, ARMREG_IP0, cfg->arch.args_reg, ((CallInfo*)cfg->arch.cinfo)->sig_cookie.offset);
4151 arm_strx (code, ARMREG_IP0, sreg1, 0);
4154 MonoInst *var = cfg->dyn_call_var;
4155 guint8 *labels [16];
4159 * sreg1 points to a DynCallArgs structure initialized by mono_arch_start_dyn_call ().
4160 * sreg2 is the function to call.
4163 g_assert (var->opcode == OP_REGOFFSET);
4165 arm_movx (code, ARMREG_LR, sreg1);
4166 arm_movx (code, ARMREG_IP1, sreg2);
4168 /* Save args buffer */
4169 code = emit_strx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4171 /* Set fp argument regs */
4172 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpargs));
4173 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4175 arm_bcc (code, ARMCOND_EQ, 0);
4176 for (i = 0; i < 8; ++i)
4177 code = emit_ldrfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4178 arm_patch_rel (labels [0], code, MONO_R_ARM64_BCC);
4180 /* Allocate callee area */
4181 code = emit_ldrx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_stackargs));
4182 arm_lslw (code, ARMREG_R0, ARMREG_R0, 3);
4183 arm_movspx (code, ARMREG_R1, ARMREG_SP);
4184 arm_subx (code, ARMREG_R1, ARMREG_R1, ARMREG_R0);
4185 arm_movspx (code, ARMREG_SP, ARMREG_R1);
4187 /* Set stack args */
4189 code = emit_ldrx (code, ARMREG_R1, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_stackargs));
4190 /* R2 = pointer into 'regs' */
4191 code = emit_imm (code, ARMREG_R2, MONO_STRUCT_OFFSET (DynCallArgs, regs) + ((PARAM_REGS + 1) * sizeof (mgreg_t)));
4192 arm_addx (code, ARMREG_R2, ARMREG_LR, ARMREG_R2);
4193 /* R3 = pointer to stack */
4194 arm_movspx (code, ARMREG_R3, ARMREG_SP);
4198 code = emit_ldrx (code, ARMREG_R5, ARMREG_R2, 0);
4199 code = emit_strx (code, ARMREG_R5, ARMREG_R3, 0);
4200 code = emit_addx_imm (code, ARMREG_R2, ARMREG_R2, sizeof (mgreg_t));
4201 code = emit_addx_imm (code, ARMREG_R3, ARMREG_R3, sizeof (mgreg_t));
4202 code = emit_subx_imm (code, ARMREG_R1, ARMREG_R1, 1);
4203 arm_patch_rel (labels [0], code, MONO_R_ARM64_B);
4204 arm_cmpw (code, ARMREG_R1, ARMREG_RZR);
4205 arm_bcc (code, ARMCOND_GT, labels [1]);
4207 /* Set argument registers + r8 */
4208 code = mono_arm_emit_load_regarray (code, 0x1ff, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, regs));
4211 arm_blrx (code, ARMREG_IP1);
4214 code = emit_ldrx (code, ARMREG_LR, var->inst_basereg, var->inst_offset);
4215 arm_strx (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res));
4216 arm_strx (code, ARMREG_R1, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, res2));
4217 /* Save fp result */
4218 code = emit_ldrw (code, ARMREG_R0, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, n_fpret));
4219 arm_cmpw (code, ARMREG_R0, ARMREG_RZR);
4221 arm_bcc (code, ARMCOND_EQ, 0);
4222 for (i = 0; i < 8; ++i)
4223 code = emit_strfpx (code, ARMREG_D0 + i, ARMREG_LR, MONO_STRUCT_OFFSET (DynCallArgs, fpregs) + (i * 8));
4224 arm_patch_rel (labels [1], code, MONO_R_ARM64_BCC);
4228 case OP_GENERIC_CLASS_INIT: {
4232 byte_offset = MONO_STRUCT_OFFSET (MonoVTable, initialized);
4234 /* Load vtable->initialized */
4235 arm_ldrsbx (code, ARMREG_IP0, sreg1, byte_offset);
4237 arm_cbnzx (code, ARMREG_IP0, 0);
4240 g_assert (sreg1 == ARMREG_R0);
4241 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4242 (gpointer)"mono_generic_class_init");
4244 mono_arm_patch (jump, code, MONO_R_ARM64_CBZ);
4249 arm_ldrx (code, ARMREG_LR, sreg1, 0);
4252 case OP_NOT_REACHED:
4255 case OP_IL_SEQ_POINT:
4256 mono_add_seq_point (cfg, bb, ins, code - cfg->native_code);
4261 case OP_COND_EXC_IC:
4262 case OP_COND_EXC_OV:
4263 case OP_COND_EXC_IOV:
4264 case OP_COND_EXC_NC:
4265 case OP_COND_EXC_INC:
4266 case OP_COND_EXC_NO:
4267 case OP_COND_EXC_INO:
4268 case OP_COND_EXC_EQ:
4269 case OP_COND_EXC_IEQ:
4270 case OP_COND_EXC_NE_UN:
4271 case OP_COND_EXC_INE_UN:
4272 case OP_COND_EXC_ILT:
4273 case OP_COND_EXC_LT:
4274 case OP_COND_EXC_ILT_UN:
4275 case OP_COND_EXC_LT_UN:
4276 case OP_COND_EXC_IGT:
4277 case OP_COND_EXC_GT:
4278 case OP_COND_EXC_IGT_UN:
4279 case OP_COND_EXC_GT_UN:
4280 case OP_COND_EXC_IGE:
4281 case OP_COND_EXC_GE:
4282 case OP_COND_EXC_IGE_UN:
4283 case OP_COND_EXC_GE_UN:
4284 case OP_COND_EXC_ILE:
4285 case OP_COND_EXC_LE:
4286 case OP_COND_EXC_ILE_UN:
4287 case OP_COND_EXC_LE_UN:
4288 code = emit_cond_exc (cfg, code, ins->opcode, ins->inst_p1);
4291 if (sreg1 != ARMREG_R0)
4292 arm_movx (code, ARMREG_R0, sreg1);
4293 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4294 (gpointer)"mono_arch_throw_exception");
4297 if (sreg1 != ARMREG_R0)
4298 arm_movx (code, ARMREG_R0, sreg1);
4299 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD,
4300 (gpointer)"mono_arch_rethrow_exception");
4302 case OP_CALL_HANDLER:
4303 mono_add_patch_info_rel (cfg, offset, MONO_PATCH_INFO_BB, ins->inst_target_bb, MONO_R_ARM64_BL);
4305 cfg->thunk_area += THUNK_SIZE;
4306 mono_cfg_add_try_hole (cfg, ins->inst_eh_block, code, bb);
4308 case OP_START_HANDLER: {
4309 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4311 /* Save caller address */
4312 code = emit_strx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4315 * Reserve a param area, see test_0_finally_param_area ().
4316 * This is needed because the param area is not set up when
4317 * we are called from EH code.
4319 if (cfg->param_area)
4320 code = emit_subx_sp_imm (code, cfg->param_area);
4324 case OP_ENDFILTER: {
4325 MonoInst *spvar = mono_find_spvar_for_region (cfg, bb->region);
4327 if (cfg->param_area)
4328 code = emit_addx_sp_imm (code, cfg->param_area);
4330 if (ins->opcode == OP_ENDFILTER && sreg1 != ARMREG_R0)
4331 arm_movx (code, ARMREG_R0, sreg1);
4333 /* Return to either after the branch in OP_CALL_HANDLER, or to the EH code */
4334 code = emit_ldrx (code, ARMREG_LR, spvar->inst_basereg, spvar->inst_offset);
4335 arm_brx (code, ARMREG_LR);
4339 if (ins->dreg != ARMREG_R0)
4340 arm_movx (code, ins->dreg, ARMREG_R0);
4342 case OP_GC_SAFE_POINT: {
4343 #if defined (USE_COOP_GC)
4346 arm_ldrx (code, ARMREG_IP1, ins->sreg1, 0);
4347 /* Call it if it is non-null */
4349 arm_cbzx (code, ARMREG_IP1, 0);
4350 code = emit_call (cfg, code, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_threads_state_poll");
4351 mono_arm_patch (buf [0], code, MONO_R_ARM64_CBZ);
4355 case OP_FILL_PROF_CALL_CTX:
4356 for (int i = 0; i < MONO_MAX_IREGS; i++)
4357 if ((MONO_ARCH_CALLEE_SAVED_REGS & (1 << i)) || i == ARMREG_SP || i == ARMREG_FP)
4358 arm_strx (code, i, ins->sreg1, MONO_STRUCT_OFFSET (MonoContext, regs) + i * sizeof (mgreg_t));
4361 g_warning ("unknown opcode %s in %s()\n", mono_inst_name (ins->opcode), __FUNCTION__);
4362 g_assert_not_reached ();
4365 if ((cfg->opt & MONO_OPT_BRANCH) && ((code - cfg->native_code - offset) > max_len)) {
4366 g_warning ("wrong maximal instruction length of instruction %s (expected %d, got %d)",
4367 mono_inst_name (ins->opcode), max_len, code - cfg->native_code - offset);
4368 g_assert_not_reached ();
4373 * If the compiled code size is larger than the bcc displacement (19 bits signed),
4374 * insert branch islands between/inside basic blocks.
4376 if (cfg->arch.cond_branch_islands)
4377 code = emit_branch_island (cfg, code, start_offset);
4379 cfg->code_len = code - cfg->native_code;
4383 emit_move_args (MonoCompile *cfg, guint8 *code)
4390 cinfo = cfg->arch.cinfo;
4392 for (i = 0; i < cinfo->nargs; ++i) {
4393 ainfo = cinfo->args + i;
4394 ins = cfg->args [i];
4396 if (ins->opcode == OP_REGVAR) {
4397 switch (ainfo->storage) {
4399 arm_movx (code, ins->dreg, ainfo->reg);
4402 switch (ainfo->slot_size) {
4405 code = emit_ldrsbx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4407 code = emit_ldrb (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4411 code = emit_ldrshx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4413 code = emit_ldrh (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4417 code = emit_ldrswx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4419 code = emit_ldrw (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4422 code = emit_ldrx (code, ins->dreg, cfg->arch.args_reg, ainfo->offset);
4427 g_assert_not_reached ();
4431 if (ainfo->storage != ArgVtypeByRef && ainfo->storage != ArgVtypeByRefOnStack)
4432 g_assert (ins->opcode == OP_REGOFFSET);
4434 switch (ainfo->storage) {
4436 /* Stack slots for arguments have size 8 */
4437 code = emit_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4440 code = emit_strfpx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4443 code = emit_strfpw (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4448 case ArgVtypeByRefOnStack:
4449 case ArgVtypeOnStack:
4451 case ArgVtypeByRef: {
4452 MonoInst *addr_arg = ins->inst_left;
4454 if (ainfo->gsharedvt) {
4455 g_assert (ins->opcode == OP_GSHAREDVT_ARG_REGOFFSET);
4456 arm_strx (code, ainfo->reg, ins->inst_basereg, ins->inst_offset);
4458 g_assert (ins->opcode == OP_VTARG_ADDR);
4459 g_assert (addr_arg->opcode == OP_REGOFFSET);
4460 arm_strx (code, ainfo->reg, addr_arg->inst_basereg, addr_arg->inst_offset);
4464 case ArgVtypeInIRegs:
4465 for (part = 0; part < ainfo->nregs; part ++) {
4466 code = emit_strx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + (part * 8));
4470 for (part = 0; part < ainfo->nregs; part ++) {
4471 if (ainfo->esize == 4)
4472 code = emit_strfpw (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4474 code = emit_strfpx (code, ainfo->reg + part, ins->inst_basereg, ins->inst_offset + ainfo->foffsets [part]);
4478 g_assert_not_reached ();
4488 * emit_store_regarray:
4490 * Emit code to store the registers in REGS into the appropriate elements of
4491 * the register array at BASEREG+OFFSET.
4493 static __attribute__ ((__warn_unused_result__)) guint8*
4494 emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4498 for (i = 0; i < 32; ++i) {
4499 if (regs & (1 << i)) {
4500 if (i + 1 < 32 && (regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4501 arm_stpx (code, i, i + 1, basereg, offset + (i * 8));
4503 } else if (i == ARMREG_SP) {
4504 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4505 arm_strx (code, ARMREG_IP1, basereg, offset + (i * 8));
4507 arm_strx (code, i, basereg, offset + (i * 8));
4515 * emit_load_regarray:
4517 * Emit code to load the registers in REGS from the appropriate elements of
4518 * the register array at BASEREG+OFFSET.
4520 static __attribute__ ((__warn_unused_result__)) guint8*
4521 emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4525 for (i = 0; i < 32; ++i) {
4526 if (regs & (1 << i)) {
4527 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4528 if (offset + (i * 8) < 500)
4529 arm_ldpx (code, i, i + 1, basereg, offset + (i * 8));
4531 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4532 code = emit_ldrx (code, i + 1, basereg, offset + ((i + 1) * 8));
4535 } else if (i == ARMREG_SP) {
4536 g_assert_not_reached ();
4538 code = emit_ldrx (code, i, basereg, offset + (i * 8));
4546 * emit_store_regset:
4548 * Emit code to store the registers in REGS into consecutive memory locations starting
4549 * at BASEREG+OFFSET.
4551 static __attribute__ ((__warn_unused_result__)) guint8*
4552 emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4557 for (i = 0; i < 32; ++i) {
4558 if (regs & (1 << i)) {
4559 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4560 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4563 } else if (i == ARMREG_SP) {
4564 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4565 arm_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4567 arm_strx (code, i, basereg, offset + (pos * 8));
4578 * Emit code to load the registers in REGS from consecutive memory locations starting
4579 * at BASEREG+OFFSET.
4581 static __attribute__ ((__warn_unused_result__)) guint8*
4582 emit_load_regset (guint8 *code, guint64 regs, int basereg, int offset)
4587 for (i = 0; i < 32; ++i) {
4588 if (regs & (1 << i)) {
4589 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4590 arm_ldpx (code, i, i + 1, basereg, offset + (pos * 8));
4593 } else if (i == ARMREG_SP) {
4594 g_assert_not_reached ();
4596 arm_ldrx (code, i, basereg, offset + (pos * 8));
4604 __attribute__ ((__warn_unused_result__)) guint8*
4605 mono_arm_emit_load_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4607 return emit_load_regarray (code, regs, basereg, offset);
4610 __attribute__ ((__warn_unused_result__)) guint8*
4611 mono_arm_emit_store_regarray (guint8 *code, guint64 regs, int basereg, int offset)
4613 return emit_store_regarray (code, regs, basereg, offset);
4616 __attribute__ ((__warn_unused_result__)) guint8*
4617 mono_arm_emit_store_regset (guint8 *code, guint64 regs, int basereg, int offset)
4619 return emit_store_regset (code, regs, basereg, offset);
4622 /* Same as emit_store_regset, but emit unwind info too */
4623 /* CFA_OFFSET is the offset between the CFA and basereg */
4624 static __attribute__ ((__warn_unused_result__)) guint8*
4625 emit_store_regset_cfa (MonoCompile *cfg, guint8 *code, guint64 regs, int basereg, int offset, int cfa_offset, guint64 no_cfa_regset)
4627 int i, j, pos, nregs;
4628 guint32 cfa_regset = regs & ~no_cfa_regset;
4631 for (i = 0; i < 32; ++i) {
4633 if (regs & (1 << i)) {
4634 if ((regs & (1 << (i + 1))) && (i + 1 != ARMREG_SP)) {
4636 arm_stpx (code, i, i + 1, basereg, offset + (pos * 8));
4638 code = emit_strx (code, i, basereg, offset + (pos * 8));
4639 code = emit_strx (code, i + 1, basereg, offset + (pos * 8) + 8);
4642 } else if (i == ARMREG_SP) {
4643 arm_movspx (code, ARMREG_IP1, ARMREG_SP);
4644 code = emit_strx (code, ARMREG_IP1, basereg, offset + (pos * 8));
4646 code = emit_strx (code, i, basereg, offset + (pos * 8));
4649 for (j = 0; j < nregs; ++j) {
4650 if (cfa_regset & (1 << (i + j)))
4651 mono_emit_unwind_op_offset (cfg, code, i + j, (- cfa_offset) + offset + ((pos + j) * 8));
4664 * Emit code to initialize an LMF structure at LMF_OFFSET.
4668 emit_setup_lmf (MonoCompile *cfg, guint8 *code, gint32 lmf_offset, int cfa_offset)
4671 * The LMF should contain all the state required to be able to reconstruct the machine state
4672 * at the current point of execution. Since the LMF is only read during EH, only callee
4673 * saved etc. registers need to be saved.
4674 * FIXME: Save callee saved fp regs, JITted code doesn't use them, but native code does, and they
4675 * need to be restored during EH.
4679 arm_adrx (code, ARMREG_LR, code);
4680 code = emit_strx (code, ARMREG_LR, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, pc));
4681 /* gregs + fp + sp */
4682 /* Don't emit unwind info for sp/fp, they are already handled in the prolog */
4683 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_LMF_REGS, ARMREG_FP, lmf_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs), cfa_offset, (1 << ARMREG_FP) | (1 << ARMREG_SP));
4689 mono_arch_emit_prolog (MonoCompile *cfg)
4691 MonoMethod *method = cfg->method;
4692 MonoMethodSignature *sig;
4695 int cfa_offset, max_offset;
4697 sig = mono_method_signature (method);
4698 cfg->code_size = 256 + sig->param_count * 64;
4699 code = cfg->native_code = g_malloc (cfg->code_size);
4701 /* This can be unaligned */
4702 cfg->stack_offset = ALIGN_TO (cfg->stack_offset, MONO_ARCH_FRAME_ALIGNMENT);
4708 mono_emit_unwind_op_def_cfa (cfg, code, ARMREG_SP, 0);
4711 if (arm_is_ldpx_imm (-cfg->stack_offset)) {
4712 arm_stpx_pre (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, -cfg->stack_offset);
4714 /* sp -= cfg->stack_offset */
4715 /* This clobbers ip0/ip1 */
4716 code = emit_subx_sp_imm (code, cfg->stack_offset);
4717 arm_stpx (code, ARMREG_FP, ARMREG_LR, ARMREG_SP, 0);
4719 cfa_offset += cfg->stack_offset;
4720 mono_emit_unwind_op_def_cfa_offset (cfg, code, cfa_offset);
4721 mono_emit_unwind_op_offset (cfg, code, ARMREG_FP, (- cfa_offset) + 0);
4722 mono_emit_unwind_op_offset (cfg, code, ARMREG_LR, (- cfa_offset) + 8);
4723 arm_movspx (code, ARMREG_FP, ARMREG_SP);
4724 mono_emit_unwind_op_def_cfa_reg (cfg, code, ARMREG_FP);
4725 if (cfg->param_area) {
4726 /* The param area is below the frame pointer */
4727 code = emit_subx_sp_imm (code, cfg->param_area);
4730 if (cfg->method->save_lmf) {
4731 code = emit_setup_lmf (cfg, code, cfg->lmf_var->inst_offset, cfa_offset);
4734 code = emit_store_regset_cfa (cfg, code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset, cfa_offset, 0);
4737 /* Setup args reg */
4738 if (cfg->arch.args_reg) {
4739 /* The register was already saved above */
4740 code = emit_addx_imm (code, cfg->arch.args_reg, ARMREG_FP, cfg->stack_offset);
4743 /* Save return area addr received in R8 */
4744 if (cfg->vret_addr) {
4745 MonoInst *ins = cfg->vret_addr;
4747 g_assert (ins->opcode == OP_REGOFFSET);
4748 code = emit_strx (code, ARMREG_R8, ins->inst_basereg, ins->inst_offset);
4751 /* Save mrgctx received in MONO_ARCH_RGCTX_REG */
4752 if (cfg->rgctx_var) {
4753 MonoInst *ins = cfg->rgctx_var;
4755 g_assert (ins->opcode == OP_REGOFFSET);
4757 code = emit_strx (code, MONO_ARCH_RGCTX_REG, ins->inst_basereg, ins->inst_offset);
4761 * Move arguments to their registers/stack locations.
4763 code = emit_move_args (cfg, code);
4765 /* Initialize seq_point_info_var */
4766 if (cfg->arch.seq_point_info_var) {
4767 MonoInst *ins = cfg->arch.seq_point_info_var;
4769 /* Initialize the variable from a GOT slot */
4770 code = emit_aotconst (cfg, code, ARMREG_IP0, MONO_PATCH_INFO_SEQ_POINT_INFO, cfg->method);
4771 g_assert (ins->opcode == OP_REGOFFSET);
4772 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4774 /* Initialize ss_tramp_var */
4775 ins = cfg->arch.ss_tramp_var;
4776 g_assert (ins->opcode == OP_REGOFFSET);
4778 code = emit_ldrx (code, ARMREG_IP1, ARMREG_IP0, MONO_STRUCT_OFFSET (SeqPointInfo, ss_tramp_addr));
4779 code = emit_strx (code, ARMREG_IP1, ins->inst_basereg, ins->inst_offset);
4783 if (cfg->arch.ss_tramp_var) {
4784 /* Initialize ss_tramp_var */
4785 ins = cfg->arch.ss_tramp_var;
4786 g_assert (ins->opcode == OP_REGOFFSET);
4788 code = emit_imm64 (code, ARMREG_IP0, (guint64)&ss_trampoline);
4789 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4792 if (cfg->arch.bp_tramp_var) {
4793 /* Initialize bp_tramp_var */
4794 ins = cfg->arch.bp_tramp_var;
4795 g_assert (ins->opcode == OP_REGOFFSET);
4797 code = emit_imm64 (code, ARMREG_IP0, (guint64)bp_trampoline);
4798 code = emit_strx (code, ARMREG_IP0, ins->inst_basereg, ins->inst_offset);
4803 if (cfg->opt & MONO_OPT_BRANCH) {
4804 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4806 bb->max_offset = max_offset;
4808 MONO_BB_FOR_EACH_INS (bb, ins) {
4809 max_offset += ((guint8 *)ins_get_spec (ins->opcode))[MONO_INST_LEN];
4813 if (max_offset > 0x3ffff * 4)
4814 cfg->arch.cond_branch_islands = TRUE;
4820 realloc_code (MonoCompile *cfg, int size)
4822 while (cfg->code_len + size > (cfg->code_size - 16)) {
4823 cfg->code_size *= 2;
4824 cfg->native_code = g_realloc (cfg->native_code, cfg->code_size);
4825 cfg->stat_code_reallocs++;
4827 return cfg->native_code + cfg->code_len;
4831 mono_arch_emit_epilog (MonoCompile *cfg)
4834 int max_epilog_size;
4838 max_epilog_size = 16 + 20*4;
4839 code = realloc_code (cfg, max_epilog_size);
4841 if (cfg->method->save_lmf) {
4842 code = mono_arm_emit_load_regarray (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->lmf_var->inst_offset + MONO_STRUCT_OFFSET (MonoLMF, gregs) - (MONO_ARCH_FIRST_LMF_REG * 8));
4845 code = emit_load_regset (code, MONO_ARCH_CALLEE_SAVED_REGS & cfg->used_int_regs, ARMREG_FP, cfg->arch.saved_gregs_offset);
4848 /* Load returned vtypes into registers if needed */
4849 cinfo = cfg->arch.cinfo;
4850 switch (cinfo->ret.storage) {
4851 case ArgVtypeInIRegs: {
4852 MonoInst *ins = cfg->ret;
4854 for (i = 0; i < cinfo->ret.nregs; ++i)
4855 code = emit_ldrx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + (i * 8));
4859 MonoInst *ins = cfg->ret;
4861 for (i = 0; i < cinfo->ret.nregs; ++i) {
4862 if (cinfo->ret.esize == 4)
4863 code = emit_ldrfpw (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
4865 code = emit_ldrfpx (code, cinfo->ret.reg + i, ins->inst_basereg, ins->inst_offset + cinfo->ret.foffsets [i]);
4874 code = mono_arm_emit_destroy_frame (code, cfg->stack_offset, ((1 << ARMREG_IP0) | (1 << ARMREG_IP1)));
4876 arm_retx (code, ARMREG_LR);
4878 g_assert (code - (cfg->native_code + cfg->code_len) < max_epilog_size);
4880 cfg->code_len = code - cfg->native_code;
4884 mono_arch_emit_exceptions (MonoCompile *cfg)
4887 MonoClass *exc_class;
4889 guint8* exc_throw_pos [MONO_EXC_INTRINS_NUM];
4890 guint8 exc_throw_found [MONO_EXC_INTRINS_NUM];
4891 int i, id, size = 0;
4893 for (i = 0; i < MONO_EXC_INTRINS_NUM; i++) {
4894 exc_throw_pos [i] = NULL;
4895 exc_throw_found [i] = 0;
4898 for (ji = cfg->patch_info; ji; ji = ji->next) {
4899 if (ji->type == MONO_PATCH_INFO_EXC) {
4900 i = mini_exception_id_by_name (ji->data.target);
4901 if (!exc_throw_found [i]) {
4903 exc_throw_found [i] = TRUE;
4908 code = realloc_code (cfg, size);
4910 /* Emit code to raise corlib exceptions */
4911 for (ji = cfg->patch_info; ji; ji = ji->next) {
4912 if (ji->type != MONO_PATCH_INFO_EXC)
4915 ip = cfg->native_code + ji->ip.i;
4917 id = mini_exception_id_by_name (ji->data.target);
4919 if (exc_throw_pos [id]) {
4920 /* ip points to the bcc () in OP_COND_EXC_... */
4921 arm_patch_rel (ip, exc_throw_pos [id], ji->relocation);
4922 ji->type = MONO_PATCH_INFO_NONE;
4926 exc_throw_pos [id] = code;
4927 arm_patch_rel (ip, code, ji->relocation);
4929 /* We are being branched to from the code generated by emit_cond_exc (), the pc is in ip1 */
4931 /* r0 = type token */
4932 exc_class = mono_class_load_from_name (mono_defaults.corlib, "System", ji->data.name);
4933 code = emit_imm (code, ARMREG_R0, exc_class->type_token - MONO_TOKEN_TYPE_DEF);
4935 arm_movx (code, ARMREG_R1, ARMREG_IP1);
4936 /* Branch to the corlib exception throwing trampoline */
4937 ji->ip.i = code - cfg->native_code;
4938 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
4939 ji->data.name = "mono_arch_throw_corlib_exception";
4940 ji->relocation = MONO_R_ARM64_BL;
4942 cfg->thunk_area += THUNK_SIZE;
4945 cfg->code_len = code - cfg->native_code;
4947 g_assert (cfg->code_len < cfg->code_size);
4951 mono_arch_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args)
4957 mono_arch_get_patch_offset (guint8 *code)
4963 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
4964 gpointer fail_tramp)
4966 int i, buf_len, imt_reg;
4970 printf ("building IMT trampoline for class %s %s entries %d code size %d code at %p end %p vtable %p\n", vtable->klass->name_space, vtable->klass->name, count, size, start, ((guint8*)start) + size, vtable);
4971 for (i = 0; i < count; ++i) {
4972 MonoIMTCheckItem *item = imt_entries [i];
4973 printf ("method %d (%p) %s vtable slot %p is_equals %d chunk size %d\n", i, item->key, item->key->name, &vtable->vtable [item->value.vtable_slot], item->is_equals, item->chunk_size);
4978 for (i = 0; i < count; ++i) {
4979 MonoIMTCheckItem *item = imt_entries [i];
4980 if (item->is_equals) {
4981 gboolean fail_case = !item->check_target_idx && fail_tramp;
4983 if (item->check_target_idx || fail_case) {
4984 if (!item->compare_done || fail_case) {
4985 buf_len += 4 * 4 + 4;
4988 if (item->has_target_code) {
5005 buf = mono_method_alloc_generic_virtual_trampoline (domain, buf_len);
5007 buf = mono_domain_code_reserve (domain, buf_len);
5011 * We are called by JITted code, which passes in the IMT argument in
5012 * MONO_ARCH_RGCTX_REG (r27). We need to preserve all caller saved regs
5015 imt_reg = MONO_ARCH_RGCTX_REG;
5016 for (i = 0; i < count; ++i) {
5017 MonoIMTCheckItem *item = imt_entries [i];
5019 item->code_target = code;
5021 if (item->is_equals) {
5023 * Check the imt argument against item->key, if equals, jump to either
5024 * item->value.target_code or to vtable [item->value.vtable_slot].
5025 * If fail_tramp is set, jump to it if not-equals.
5027 gboolean fail_case = !item->check_target_idx && fail_tramp;
5029 if (item->check_target_idx || fail_case) {
5030 /* Compare imt_reg with item->key */
5031 if (!item->compare_done || fail_case) {
5032 // FIXME: Optimize this
5033 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5034 arm_cmpx (code, imt_reg, ARMREG_IP0);
5036 item->jmp_code = code;
5037 arm_bcc (code, ARMCOND_NE, 0);
5038 /* Jump to target if equals */
5039 if (item->has_target_code) {
5040 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->value.target_code);
5041 arm_brx (code, ARMREG_IP0);
5043 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5045 code = emit_imm64 (code, ARMREG_IP0, imm);
5046 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5047 arm_brx (code, ARMREG_IP0);
5051 arm_patch_rel (item->jmp_code, code, MONO_R_ARM64_BCC);
5052 item->jmp_code = NULL;
5053 code = emit_imm64 (code, ARMREG_IP0, (guint64)fail_tramp);
5054 arm_brx (code, ARMREG_IP0);
5057 guint64 imm = (guint64)&(vtable->vtable [item->value.vtable_slot]);
5059 code = emit_imm64 (code, ARMREG_IP0, imm);
5060 arm_ldrx (code, ARMREG_IP0, ARMREG_IP0, 0);
5061 arm_brx (code, ARMREG_IP0);
5064 code = emit_imm64 (code, ARMREG_IP0, (guint64)item->key);
5065 arm_cmpx (code, imt_reg, ARMREG_IP0);
5066 item->jmp_code = code;
5067 arm_bcc (code, ARMCOND_HS, 0);
5070 /* Patch the branches */
5071 for (i = 0; i < count; ++i) {
5072 MonoIMTCheckItem *item = imt_entries [i];
5073 if (item->jmp_code && item->check_target_idx)
5074 arm_patch_rel (item->jmp_code, imt_entries [item->check_target_idx]->code_target, MONO_R_ARM64_BCC);
5077 g_assert ((code - buf) < buf_len);
5079 mono_arch_flush_icache (buf, code - buf);
5085 mono_arch_get_trampolines (gboolean aot)
5087 return mono_arm_get_exception_trampolines (aot);
5090 #else /* DISABLE_JIT */
5093 mono_arch_build_imt_trampoline (MonoVTable *vtable, MonoDomain *domain, MonoIMTCheckItem **imt_entries, int count,
5094 gpointer fail_tramp)
5096 g_assert_not_reached ();
5100 #endif /* !DISABLE_JIT */
5102 #ifdef MONO_ARCH_SOFT_DEBUG_SUPPORTED
5105 mono_arch_set_breakpoint (MonoJitInfo *ji, guint8 *ip)
5108 guint32 native_offset = ip - (guint8*)ji->code_start;
5111 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5113 g_assert (native_offset % 4 == 0);
5114 g_assert (info->bp_addrs [native_offset / 4] == 0);
5115 info->bp_addrs [native_offset / 4] = mini_get_breakpoint_trampoline ();
5117 /* ip points to an ldrx */
5119 arm_blrx (code, ARMREG_IP0);
5120 mono_arch_flush_icache (ip, code - ip);
5125 mono_arch_clear_breakpoint (MonoJitInfo *ji, guint8 *ip)
5130 guint32 native_offset = ip - (guint8*)ji->code_start;
5131 SeqPointInfo *info = mono_arch_get_seq_point_info (mono_domain_get (), ji->code_start);
5133 g_assert (native_offset % 4 == 0);
5134 info->bp_addrs [native_offset / 4] = NULL;
5136 /* ip points to an ldrx */
5139 mono_arch_flush_icache (ip, code - ip);
5144 mono_arch_start_single_stepping (void)
5146 ss_trampoline = mini_get_single_step_trampoline ();
5150 mono_arch_stop_single_stepping (void)
5152 ss_trampoline = NULL;
5156 mono_arch_is_single_step_event (void *info, void *sigctx)
5158 /* We use soft breakpoints on arm64 */
5163 mono_arch_is_breakpoint_event (void *info, void *sigctx)
5165 /* We use soft breakpoints on arm64 */
5170 mono_arch_skip_breakpoint (MonoContext *ctx, MonoJitInfo *ji)
5172 g_assert_not_reached ();
5176 mono_arch_skip_single_step (MonoContext *ctx)
5178 g_assert_not_reached ();
5182 mono_arch_get_seq_point_info (MonoDomain *domain, guint8 *code)
5187 // FIXME: Add a free function
5189 mono_domain_lock (domain);
5190 info = g_hash_table_lookup (domain_jit_info (domain)->arch_seq_points,
5192 mono_domain_unlock (domain);
5195 ji = mono_jit_info_table_find (domain, (char*)code);
5198 info = g_malloc0 (sizeof (SeqPointInfo) + (ji->code_size / 4) * sizeof(guint8*));
5200 info->ss_tramp_addr = &ss_trampoline;
5202 mono_domain_lock (domain);
5203 g_hash_table_insert (domain_jit_info (domain)->arch_seq_points,
5205 mono_domain_unlock (domain);
5212 mono_arch_init_lmf_ext (MonoLMFExt *ext, gpointer prev_lmf)
5214 ext->lmf.previous_lmf = prev_lmf;
5215 /* Mark that this is a MonoLMFExt */
5216 ext->lmf.previous_lmf = (gpointer)(((gssize)ext->lmf.previous_lmf) | 2);
5217 ext->lmf.gregs [MONO_ARCH_LMF_REG_SP] = (gssize)ext;
5220 #endif /* MONO_ARCH_SOFT_DEBUG_SUPPORTED */
5223 mono_arch_opcode_supported (int opcode)
5226 case OP_ATOMIC_ADD_I4:
5227 case OP_ATOMIC_ADD_I8:
5228 case OP_ATOMIC_EXCHANGE_I4:
5229 case OP_ATOMIC_EXCHANGE_I8:
5230 case OP_ATOMIC_CAS_I4:
5231 case OP_ATOMIC_CAS_I8:
5232 case OP_ATOMIC_LOAD_I1:
5233 case OP_ATOMIC_LOAD_I2:
5234 case OP_ATOMIC_LOAD_I4:
5235 case OP_ATOMIC_LOAD_I8:
5236 case OP_ATOMIC_LOAD_U1:
5237 case OP_ATOMIC_LOAD_U2:
5238 case OP_ATOMIC_LOAD_U4:
5239 case OP_ATOMIC_LOAD_U8:
5240 case OP_ATOMIC_LOAD_R4:
5241 case OP_ATOMIC_LOAD_R8:
5242 case OP_ATOMIC_STORE_I1:
5243 case OP_ATOMIC_STORE_I2:
5244 case OP_ATOMIC_STORE_I4:
5245 case OP_ATOMIC_STORE_I8:
5246 case OP_ATOMIC_STORE_U1:
5247 case OP_ATOMIC_STORE_U2:
5248 case OP_ATOMIC_STORE_U4:
5249 case OP_ATOMIC_STORE_U8:
5250 case OP_ATOMIC_STORE_R4:
5251 case OP_ATOMIC_STORE_R8:
5259 mono_arch_get_call_info (MonoMemPool *mp, MonoMethodSignature *sig)
5261 return get_call_info (mp, sig);