2 * mini-llvm.c: llvm "Backend" for the mono JIT
4 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
5 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
9 #include <mono/metadata/debug-helpers.h>
10 #include <mono/metadata/debug-mono-symfile.h>
11 #include <mono/metadata/mempool-internals.h>
12 #include <mono/utils/mono-tls.h>
13 #include <mono/utils/mono-dl.h>
14 #include <mono/utils/mono-time.h>
15 #include <mono/utils/freebsd-dwarf.h>
17 #ifndef __STDC_LIMIT_MACROS
18 #define __STDC_LIMIT_MACROS
20 #ifndef __STDC_CONSTANT_MACROS
21 #define __STDC_CONSTANT_MACROS
24 #include "llvm-c/Core.h"
25 #include "llvm-c/ExecutionEngine.h"
26 #include "llvm-c/BitWriter.h"
27 #include "llvm-c/Analysis.h"
29 #include "mini-llvm-cpp.h"
34 extern void *memset(void *, int, size_t);
35 void bzero (void *to, size_t count) { memset (to, 0, count); }
40 * Information associated by mono with LLVM modules.
44 LLVMValueRef throw, rethrow, throw_corlib_exception;
45 GHashTable *llvm_types;
47 const char *got_symbol;
48 GHashTable *plt_entries;
49 GHashTable *plt_entries_ji;
50 GHashTable *method_to_lmethod;
55 GPtrArray *subprogram_mds;
57 LLVMExecutionEngineRef ee;
58 gboolean external_symbols;
64 * Information associated by the backend with mono basic blocks.
67 LLVMBasicBlockRef bblock, end_bblock;
68 LLVMValueRef finally_ind;
69 gboolean added, invoke_target;
71 * If this bblock is the start of a finally clause, this is a list of bblocks it
72 * needs to branch to in ENDFINALLY.
74 GSList *call_handler_return_bbs;
76 * If this bblock is the start of a finally clause, this is the bblock that
77 * CALL_HANDLER needs to branch to.
79 LLVMBasicBlockRef call_handler_target_bb;
80 /* The list of switch statements generated by ENDFINALLY instructions */
81 GSList *endfinally_switch_ins_list;
86 * Structure containing emit state
91 /* Maps method names to the corresponding LLVMValueRef */
92 GHashTable *emitted_method_decls;
96 MonoLLVMModule *lmodule;
99 int sindex, default_index, ex_index;
100 LLVMBuilderRef builder;
101 LLVMValueRef *values, *addresses;
102 MonoType **vreg_cli_types;
104 MonoMethodSignature *sig;
106 GHashTable *region_to_handler;
107 LLVMBuilderRef alloca_builder;
108 LLVMValueRef last_alloca;
109 LLVMValueRef rgctx_arg;
110 LLVMTypeRef *vreg_types;
112 gboolean *unreachable;
114 LLVMValueRef imt_rgctx_loc;
115 GHashTable *llvm_types;
117 MonoDebugMethodInfo *minfo;
124 MonoBasicBlock *in_bb;
129 * Instruction metadata
130 * This is the same as ins_info, but LREG != IREG.
138 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
139 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
146 /* keep in sync with the enum in mini.h */
149 #include "mini-ops.h"
154 #if SIZEOF_VOID_P == 4
155 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
157 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
160 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
163 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
165 #define TRACE_FAILURE(msg)
169 #define IS_TARGET_X86 1
171 #define IS_TARGET_X86 0
175 #define IS_TARGET_AMD64 1
177 #define IS_TARGET_AMD64 0
180 #define LLVM_FAILURE(ctx, reason) do { \
181 TRACE_FAILURE (reason); \
182 (ctx)->cfg->exception_message = g_strdup (reason); \
183 (ctx)->cfg->disable_llvm = TRUE; \
187 #define CHECK_FAILURE(ctx) do { \
188 if ((ctx)->cfg->disable_llvm) \
192 static LLVMIntPredicate cond_to_llvm_cond [] = {
205 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
218 static MonoNativeTlsKey current_cfg_tls_id;
220 static MonoLLVMModule aot_module;
221 static int memset_param_count, memcpy_param_count;
222 static const char *memset_func_name;
223 static const char *memcpy_func_name;
225 static void init_jit_module (MonoDomain *domain);
227 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
228 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
229 static void emit_dbg_info (MonoLLVMModule *lmodule, const char *filename, const char *cu_name);
234 * The LLVM type with width == sizeof (gpointer)
239 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
245 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
251 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
257 * Return the size of the LLVM representation of the vtype T.
260 get_vtype_size (MonoType *t)
264 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
266 while (size < 2 * sizeof (gpointer) && mono_is_power_of_two (size) == -1)
273 * simd_class_to_llvm_type:
275 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
278 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
280 if (!strcmp (klass->name, "Vector2d")) {
281 return LLVMVectorType (LLVMDoubleType (), 2);
282 } else if (!strcmp (klass->name, "Vector2l")) {
283 return LLVMVectorType (LLVMInt64Type (), 2);
284 } else if (!strcmp (klass->name, "Vector2ul")) {
285 return LLVMVectorType (LLVMInt64Type (), 2);
286 } else if (!strcmp (klass->name, "Vector4i")) {
287 return LLVMVectorType (LLVMInt32Type (), 4);
288 } else if (!strcmp (klass->name, "Vector4ui")) {
289 return LLVMVectorType (LLVMInt32Type (), 4);
290 } else if (!strcmp (klass->name, "Vector4f")) {
291 return LLVMVectorType (LLVMFloatType (), 4);
292 } else if (!strcmp (klass->name, "Vector8s")) {
293 return LLVMVectorType (LLVMInt16Type (), 8);
294 } else if (!strcmp (klass->name, "Vector8us")) {
295 return LLVMVectorType (LLVMInt16Type (), 8);
296 } else if (!strcmp (klass->name, "Vector16sb")) {
297 return LLVMVectorType (LLVMInt8Type (), 16);
298 } else if (!strcmp (klass->name, "Vector16b")) {
299 return LLVMVectorType (LLVMInt8Type (), 16);
301 printf ("%s\n", klass->name);
307 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
308 static inline G_GNUC_UNUSED LLVMTypeRef
309 type_to_simd_type (int type)
313 return LLVMVectorType (LLVMInt8Type (), 16);
315 return LLVMVectorType (LLVMInt16Type (), 8);
317 return LLVMVectorType (LLVMInt32Type (), 4);
319 return LLVMVectorType (LLVMInt64Type (), 2);
321 return LLVMVectorType (LLVMDoubleType (), 2);
323 return LLVMVectorType (LLVMFloatType (), 4);
325 g_assert_not_reached ();
331 create_llvm_type_for_type (MonoClass *klass)
333 int i, size, nfields, esize;
334 LLVMTypeRef *eltypes;
339 t = &klass->byval_arg;
341 if (mini_type_is_hfa (t, &nfields, &esize)) {
343 * This is needed on arm64 where HFAs are returned in
347 eltypes = g_new (LLVMTypeRef, size);
348 for (i = 0; i < size; ++i)
349 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
351 size = get_vtype_size (t);
353 eltypes = g_new (LLVMTypeRef, size);
354 for (i = 0; i < size; ++i)
355 eltypes [i] = LLVMInt8Type ();
358 name = mono_type_full_name (&klass->byval_arg);
359 ltype = LLVMStructCreateNamed (LLVMGetGlobalContext (), name);
360 LLVMStructSetBody (ltype, eltypes, size, FALSE);
370 * Return the LLVM type corresponding to T.
373 type_to_llvm_type (EmitContext *ctx, MonoType *t)
375 t = mini_replace_type (t);
378 return LLVMPointerType (LLVMInt8Type (), 0);
381 return LLVMVoidType ();
383 return LLVMInt8Type ();
385 return LLVMInt16Type ();
387 return LLVMInt32Type ();
389 return LLVMInt8Type ();
391 return LLVMInt16Type ();
393 return LLVMInt32Type ();
394 case MONO_TYPE_BOOLEAN:
395 return LLVMInt8Type ();
398 return LLVMInt64Type ();
400 return LLVMInt16Type ();
402 return LLVMFloatType ();
404 return LLVMDoubleType ();
407 return IntPtrType ();
408 case MONO_TYPE_OBJECT:
409 case MONO_TYPE_CLASS:
410 case MONO_TYPE_ARRAY:
411 case MONO_TYPE_SZARRAY:
412 case MONO_TYPE_STRING:
414 return ObjRefType ();
417 /* Because of generic sharing */
418 return ObjRefType ();
419 case MONO_TYPE_GENERICINST:
420 if (!mono_type_generic_inst_is_valuetype (t))
421 return ObjRefType ();
423 case MONO_TYPE_VALUETYPE:
424 case MONO_TYPE_TYPEDBYREF: {
428 klass = mono_class_from_mono_type (t);
430 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
431 return simd_class_to_llvm_type (ctx, klass);
434 return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
436 ltype = g_hash_table_lookup (ctx->lmodule->llvm_types, klass);
438 ltype = create_llvm_type_for_type (klass);
439 g_hash_table_insert (ctx->lmodule->llvm_types, klass, ltype);
445 printf ("X: %d\n", t->type);
446 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
447 ctx->cfg->disable_llvm = TRUE;
455 * Return whenever T is an unsigned int type.
458 type_is_unsigned (EmitContext *ctx, MonoType *t)
475 * type_to_llvm_arg_type:
477 * Same as type_to_llvm_type, but treat i8/i16 as i32.
480 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
482 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
485 * This works on all abis except arm64/ios which passes multiple
486 * arguments in one stack slot.
489 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
491 * LLVM generates code which only sets the lower bits, while JITted
492 * code expects all the bits to be set.
494 ptype = LLVMInt32Type ();
502 * llvm_type_to_stack_type:
504 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
507 static G_GNUC_UNUSED LLVMTypeRef
508 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
512 if (type == LLVMInt8Type ())
513 return LLVMInt32Type ();
514 else if (type == LLVMInt16Type ())
515 return LLVMInt32Type ();
516 else if (!cfg->r4fp && type == LLVMFloatType ())
517 return LLVMDoubleType ();
523 * regtype_to_llvm_type:
525 * Return the LLVM type corresponding to the regtype C used in instruction
529 regtype_to_llvm_type (char c)
533 return LLVMInt32Type ();
535 return LLVMInt64Type ();
537 return LLVMDoubleType ();
546 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
549 op_to_llvm_type (int opcode)
554 return LLVMInt8Type ();
557 return LLVMInt8Type ();
560 return LLVMInt16Type ();
563 return LLVMInt16Type ();
566 return LLVMInt32Type ();
569 return LLVMInt32Type ();
571 return LLVMInt64Type ();
573 return LLVMFloatType ();
575 return LLVMDoubleType ();
577 return LLVMInt64Type ();
579 return LLVMInt32Type ();
581 return LLVMInt64Type ();
586 return LLVMInt8Type ();
591 return LLVMInt16Type ();
594 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
601 return LLVMInt32Type ();
608 return LLVMInt64Type ();
610 printf ("%s\n", mono_inst_name (opcode));
611 g_assert_not_reached ();
617 * load_store_to_llvm_type:
619 * Return the size/sign/zero extension corresponding to the load/store opcode
623 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
629 case OP_LOADI1_MEMBASE:
630 case OP_STOREI1_MEMBASE_REG:
631 case OP_STOREI1_MEMBASE_IMM:
632 case OP_ATOMIC_LOAD_I1:
633 case OP_ATOMIC_STORE_I1:
636 return LLVMInt8Type ();
637 case OP_LOADU1_MEMBASE:
639 case OP_ATOMIC_LOAD_U1:
640 case OP_ATOMIC_STORE_U1:
643 return LLVMInt8Type ();
644 case OP_LOADI2_MEMBASE:
645 case OP_STOREI2_MEMBASE_REG:
646 case OP_STOREI2_MEMBASE_IMM:
647 case OP_ATOMIC_LOAD_I2:
648 case OP_ATOMIC_STORE_I2:
651 return LLVMInt16Type ();
652 case OP_LOADU2_MEMBASE:
654 case OP_ATOMIC_LOAD_U2:
655 case OP_ATOMIC_STORE_U2:
658 return LLVMInt16Type ();
659 case OP_LOADI4_MEMBASE:
660 case OP_LOADU4_MEMBASE:
663 case OP_STOREI4_MEMBASE_REG:
664 case OP_STOREI4_MEMBASE_IMM:
665 case OP_ATOMIC_LOAD_I4:
666 case OP_ATOMIC_STORE_I4:
667 case OP_ATOMIC_LOAD_U4:
668 case OP_ATOMIC_STORE_U4:
670 return LLVMInt32Type ();
671 case OP_LOADI8_MEMBASE:
673 case OP_STOREI8_MEMBASE_REG:
674 case OP_STOREI8_MEMBASE_IMM:
675 case OP_ATOMIC_LOAD_I8:
676 case OP_ATOMIC_STORE_I8:
677 case OP_ATOMIC_LOAD_U8:
678 case OP_ATOMIC_STORE_U8:
680 return LLVMInt64Type ();
681 case OP_LOADR4_MEMBASE:
682 case OP_STORER4_MEMBASE_REG:
683 case OP_ATOMIC_LOAD_R4:
684 case OP_ATOMIC_STORE_R4:
686 return LLVMFloatType ();
687 case OP_LOADR8_MEMBASE:
688 case OP_STORER8_MEMBASE_REG:
689 case OP_ATOMIC_LOAD_R8:
690 case OP_ATOMIC_STORE_R8:
692 return LLVMDoubleType ();
693 case OP_LOAD_MEMBASE:
695 case OP_STORE_MEMBASE_REG:
696 case OP_STORE_MEMBASE_IMM:
697 *size = sizeof (gpointer);
698 return IntPtrType ();
700 g_assert_not_reached ();
708 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
711 ovf_op_to_intrins (int opcode)
715 return "llvm.sadd.with.overflow.i32";
717 return "llvm.uadd.with.overflow.i32";
719 return "llvm.ssub.with.overflow.i32";
721 return "llvm.usub.with.overflow.i32";
723 return "llvm.smul.with.overflow.i32";
725 return "llvm.umul.with.overflow.i32";
727 return "llvm.sadd.with.overflow.i64";
729 return "llvm.uadd.with.overflow.i64";
731 return "llvm.ssub.with.overflow.i64";
733 return "llvm.usub.with.overflow.i64";
735 return "llvm.smul.with.overflow.i64";
737 return "llvm.umul.with.overflow.i64";
739 g_assert_not_reached ();
745 simd_op_to_intrins (int opcode)
748 #if defined(TARGET_X86) || defined(TARGET_AMD64)
750 return "llvm.x86.sse2.min.pd";
752 return "llvm.x86.sse.min.ps";
754 return "llvm.x86.sse41.pminud";
756 return "llvm.x86.sse41.pminuw";
758 return "llvm.x86.sse2.pminu.b";
760 return "llvm.x86.sse2.pmins.w";
762 return "llvm.x86.sse2.max.pd";
764 return "llvm.x86.sse.max.ps";
766 return "llvm.x86.sse3.hadd.pd";
768 return "llvm.x86.sse3.hadd.ps";
770 return "llvm.x86.sse3.hsub.pd";
772 return "llvm.x86.sse3.hsub.ps";
774 return "llvm.x86.sse41.pmaxud";
776 return "llvm.x86.sse41.pmaxuw";
778 return "llvm.x86.sse2.pmaxu.b";
780 return "llvm.x86.sse3.addsub.ps";
782 return "llvm.x86.sse3.addsub.pd";
783 case OP_EXTRACT_MASK:
784 return "llvm.x86.sse2.pmovmskb.128";
787 return "llvm.x86.sse2.psrli.w";
790 return "llvm.x86.sse2.psrli.d";
793 return "llvm.x86.sse2.psrli.q";
796 return "llvm.x86.sse2.pslli.w";
799 return "llvm.x86.sse2.pslli.d";
802 return "llvm.x86.sse2.pslli.q";
805 return "llvm.x86.sse2.psrai.w";
808 return "llvm.x86.sse2.psrai.d";
810 return "llvm.x86.sse2.padds.b";
812 return "llvm.x86.sse2.padds.w";
814 return "llvm.x86.sse2.psubs.b";
816 return "llvm.x86.sse2.psubs.w";
817 case OP_PADDB_SAT_UN:
818 return "llvm.x86.sse2.paddus.b";
819 case OP_PADDW_SAT_UN:
820 return "llvm.x86.sse2.paddus.w";
821 case OP_PSUBB_SAT_UN:
822 return "llvm.x86.sse2.psubus.b";
823 case OP_PSUBW_SAT_UN:
824 return "llvm.x86.sse2.psubus.w";
826 return "llvm.x86.sse2.pavg.b";
828 return "llvm.x86.sse2.pavg.w";
830 return "llvm.x86.sse.sqrt.ps";
832 return "llvm.x86.sse2.sqrt.pd";
834 return "llvm.x86.sse.rsqrt.ps";
836 return "llvm.x86.sse.rcp.ps";
838 return "llvm.x86.sse2.cvtdq2pd";
840 return "llvm.x86.sse2.cvtdq2ps";
842 return "llvm.x86.sse2.cvtpd2dq";
844 return "llvm.x86.sse2.cvtps2dq";
846 return "llvm.x86.sse2.cvtpd2ps";
848 return "llvm.x86.sse2.cvtps2pd";
850 return "llvm.x86.sse2.cvttpd2dq";
852 return "llvm.x86.sse2.cvttps2dq";
854 return "llvm.x86.sse.cmp.ps";
856 return "llvm.x86.sse2.cmp.pd";
858 return "llvm.x86.sse2.packsswb.128";
860 return "llvm.x86.sse2.packssdw.128";
862 return "llvm.x86.sse2.packuswb.128";
864 return "llvm.x86.sse41.packusdw";
866 return "llvm.x86.sse2.pmulh.w";
867 case OP_PMULW_HIGH_UN:
868 return "llvm.x86.sse2.pmulhu.w";
871 g_assert_not_reached ();
877 simd_op_to_llvm_type (int opcode)
879 #if defined(TARGET_X86) || defined(TARGET_AMD64)
883 return type_to_simd_type (MONO_TYPE_R8);
886 return type_to_simd_type (MONO_TYPE_I8);
889 return type_to_simd_type (MONO_TYPE_I4);
894 return type_to_simd_type (MONO_TYPE_I2);
898 return type_to_simd_type (MONO_TYPE_I1);
900 return type_to_simd_type (MONO_TYPE_R4);
903 return type_to_simd_type (MONO_TYPE_I4);
907 return type_to_simd_type (MONO_TYPE_R8);
911 return type_to_simd_type (MONO_TYPE_R4);
912 case OP_EXTRACT_MASK:
913 return type_to_simd_type (MONO_TYPE_I1);
919 return type_to_simd_type (MONO_TYPE_R4);
922 return type_to_simd_type (MONO_TYPE_R8);
924 g_assert_not_reached ();
935 * Return the LLVM basic block corresponding to BB.
937 static LLVMBasicBlockRef
938 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
940 char bb_name_buf [128];
943 if (ctx->bblocks [bb->block_num].bblock == NULL) {
944 if (bb->flags & BB_EXCEPTION_HANDLER) {
945 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
946 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
947 bb_name = bb_name_buf;
948 } else if (bb->block_num < 256) {
949 if (!ctx->lmodule->bb_names) {
950 ctx->lmodule->bb_names_len = 256;
951 ctx->lmodule->bb_names = g_new0 (char*, ctx->lmodule->bb_names_len);
953 if (!ctx->lmodule->bb_names [bb->block_num]) {
956 n = g_strdup_printf ("BB%d", bb->block_num);
957 mono_memory_barrier ();
958 ctx->lmodule->bb_names [bb->block_num] = n;
960 bb_name = ctx->lmodule->bb_names [bb->block_num];
962 sprintf (bb_name_buf, "BB%d", bb->block_num);
963 bb_name = bb_name_buf;
966 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
967 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
970 return ctx->bblocks [bb->block_num].bblock;
976 * Return the last LLVM bblock corresponding to BB.
977 * This might not be equal to the bb returned by get_bb () since we need to generate
978 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
980 static LLVMBasicBlockRef
981 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
984 return ctx->bblocks [bb->block_num].end_bblock;
987 static LLVMBasicBlockRef
988 gen_bb (EmitContext *ctx, const char *prefix)
992 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
993 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
999 * Return the target of the patch identified by TYPE and TARGET.
1002 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1006 memset (&ji, 0, sizeof (ji));
1008 ji.data.target = target;
1010 return mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
1016 * Emit code to convert the LLVM value V to DTYPE.
1019 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1021 LLVMTypeRef stype = LLVMTypeOf (v);
1023 if (stype != dtype) {
1024 gboolean ext = FALSE;
1027 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1029 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1031 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1035 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1037 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1038 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1041 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1042 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1043 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1044 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1045 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1046 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1047 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1048 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1050 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1051 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1052 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1053 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1054 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1055 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1057 if (mono_arch_is_soft_float ()) {
1058 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1059 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1060 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1061 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1064 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1065 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1068 LLVMDumpValue (LLVMConstNull (dtype));
1069 g_assert_not_reached ();
1077 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1079 return convert_full (ctx, v, dtype, FALSE);
1083 * emit_volatile_load:
1085 * If vreg is volatile, emit a load from its address.
1088 emit_volatile_load (EmitContext *ctx, int vreg)
1092 LLVMValueRef v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1093 t = ctx->vreg_cli_types [vreg];
1094 if (t && !t->byref) {
1096 * Might have to zero extend since llvm doesn't have
1099 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1100 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1101 else if (t->type == MONO_TYPE_U8)
1102 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1109 * emit_volatile_store:
1111 * If VREG is volatile, emit a store from its value to its address.
1114 emit_volatile_store (EmitContext *ctx, int vreg)
1116 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1118 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1119 g_assert (ctx->addresses [vreg]);
1120 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1126 * Maps parameter indexes in the original signature to parameter indexes
1127 * in the LLVM signature.
1130 /* The indexes of various special arguments in the LLVM signature */
1131 int vret_arg_pindex, this_arg_pindex, rgctx_arg_pindex, imt_arg_pindex;
1135 * sig_to_llvm_sig_full:
1137 * Return the LLVM signature corresponding to the mono signature SIG using the
1138 * calling convention information in CINFO. Return parameter mapping information in SINFO.
1141 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo,
1144 LLVMTypeRef ret_type;
1145 LLVMTypeRef *param_types = NULL;
1147 int i, j, pindex, vret_arg_pindex = 0;
1149 gboolean vretaddr = FALSE;
1153 memset (sinfo, 0, sizeof (LLVMSigInfo));
1155 rtype = mini_replace_type (sig->ret);
1156 ret_type = type_to_llvm_type (ctx, rtype);
1157 CHECK_FAILURE (ctx);
1159 if (cinfo && cinfo->ret.storage == LLVMArgVtypeInReg) {
1160 /* LLVM models this by returning an aggregate value */
1161 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1162 LLVMTypeRef members [2];
1164 members [0] = IntPtrType ();
1165 ret_type = LLVMStructType (members, 1, FALSE);
1167 g_assert_not_reached ();
1169 } else if (cinfo && cinfo->ret.storage == LLVMArgVtypeByVal) {
1170 /* Vtype returned normally by val */
1171 } else if (cinfo && mini_type_is_vtype (ctx->cfg, rtype)) {
1172 g_assert (cinfo->ret.storage == LLVMArgVtypeRetAddr);
1174 ret_type = LLVMVoidType ();
1177 pindexes = g_new0 (int, sig->param_count);
1178 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1180 if (cinfo && cinfo->rgctx_arg) {
1182 sinfo->rgctx_arg_pindex = pindex;
1183 param_types [pindex] = ctx->lmodule->ptr_type;
1186 if (cinfo && cinfo->imt_arg) {
1188 sinfo->imt_arg_pindex = pindex;
1189 param_types [pindex] = ctx->lmodule->ptr_type;
1193 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1194 vret_arg_pindex = pindex;
1195 if (cinfo->vret_arg_index == 1) {
1196 /* Add the slots consumed by the first argument */
1197 LLVMArgInfo *ainfo = &cinfo->args [0];
1198 switch (ainfo->storage) {
1199 case LLVMArgVtypeInReg:
1200 for (j = 0; j < 2; ++j) {
1201 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1211 sinfo->vret_arg_pindex = vret_arg_pindex;
1214 if (vretaddr && vret_arg_pindex == pindex)
1215 param_types [pindex ++] = IntPtrType ();
1218 sinfo->this_arg_pindex = pindex;
1219 param_types [pindex ++] = ThisType ();
1221 if (vretaddr && vret_arg_pindex == pindex)
1222 param_types [pindex ++] = IntPtrType ();
1223 for (i = 0; i < sig->param_count; ++i) {
1224 LLVMArgInfo *ainfo = cinfo ? &cinfo->args [i + sig->hasthis] : NULL;
1226 if (vretaddr && vret_arg_pindex == pindex)
1227 param_types [pindex ++] = IntPtrType ();
1228 pindexes [i] = pindex;
1229 if (ainfo && ainfo->storage == LLVMArgVtypeInReg) {
1230 for (j = 0; j < 2; ++j) {
1231 switch (ainfo->pair_storage [j]) {
1233 param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
1238 g_assert_not_reached ();
1241 } else if (ainfo && ainfo->storage == LLVMArgVtypeByVal) {
1242 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->params [i]);
1243 CHECK_FAILURE (ctx);
1244 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1246 } else if (ainfo && ainfo->storage == LLVMArgAsIArgs) {
1247 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1249 } else if (ainfo && ainfo->storage == LLVMArgAsFpArgs) {
1252 for (j = 0; j < ainfo->nslots; ++j)
1253 param_types [pindex + j] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1254 pindex += ainfo->nslots;
1256 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1259 if (vretaddr && vret_arg_pindex == pindex)
1260 param_types [pindex ++] = IntPtrType ();
1262 CHECK_FAILURE (ctx);
1264 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1265 g_free (param_types);
1268 sinfo->pindexes = pindexes;
1276 g_free (param_types);
1282 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1284 return sig_to_llvm_sig_full (ctx, sig, NULL, NULL);
1288 * LLVMFunctionType1:
1290 * Create an LLVM function type from the arguments.
1292 static G_GNUC_UNUSED LLVMTypeRef
1293 LLVMFunctionType1(LLVMTypeRef ReturnType,
1294 LLVMTypeRef ParamType1,
1297 LLVMTypeRef param_types [1];
1299 param_types [0] = ParamType1;
1301 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1305 * LLVMFunctionType2:
1307 * Create an LLVM function type from the arguments.
1309 static G_GNUC_UNUSED LLVMTypeRef
1310 LLVMFunctionType2(LLVMTypeRef ReturnType,
1311 LLVMTypeRef ParamType1,
1312 LLVMTypeRef ParamType2,
1315 LLVMTypeRef param_types [2];
1317 param_types [0] = ParamType1;
1318 param_types [1] = ParamType2;
1320 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1324 * LLVMFunctionType3:
1326 * Create an LLVM function type from the arguments.
1328 static G_GNUC_UNUSED LLVMTypeRef
1329 LLVMFunctionType3(LLVMTypeRef ReturnType,
1330 LLVMTypeRef ParamType1,
1331 LLVMTypeRef ParamType2,
1332 LLVMTypeRef ParamType3,
1335 LLVMTypeRef param_types [3];
1337 param_types [0] = ParamType1;
1338 param_types [1] = ParamType2;
1339 param_types [2] = ParamType3;
1341 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1347 * Create an LLVM builder and remember it so it can be freed later.
1349 static LLVMBuilderRef
1350 create_builder (EmitContext *ctx)
1352 LLVMBuilderRef builder = LLVMCreateBuilder ();
1354 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1360 get_plt_entry (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1362 char *callee_name = mono_aot_get_plt_symbol (type, data);
1363 LLVMValueRef callee;
1364 MonoJumpInfo *ji = NULL;
1369 if (ctx->cfg->compile_aot)
1370 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1371 mono_add_patch_info (ctx->cfg, 0, type, data);
1374 callee = g_hash_table_lookup (ctx->lmodule->plt_entries, callee_name);
1376 callee = LLVMAddFunction (ctx->module, callee_name, llvm_sig);
1378 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1380 g_hash_table_insert (ctx->lmodule->plt_entries, (char*)callee_name, callee);
1383 if (ctx->cfg->compile_aot) {
1384 ji = g_new0 (MonoJumpInfo, 1);
1386 ji->data.target = data;
1388 g_hash_table_insert (ctx->lmodule->plt_entries_ji, ji, callee);
1395 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
1397 MonoMethodHeader *header = cfg->header;
1398 MonoExceptionClause *clause;
1402 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
1403 return (bb->region >> 8) - 1;
1406 for (i = 0; i < header->num_clauses; ++i) {
1407 clause = &header->clauses [i];
1409 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
1417 set_metadata_flag (LLVMValueRef v, const char *flag_name)
1419 LLVMValueRef md_arg;
1422 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1423 md_arg = LLVMMDString ("mono", 4);
1424 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1428 set_invariant_load_flag (LLVMValueRef v)
1430 LLVMValueRef md_arg;
1432 const char *flag_name;
1434 // FIXME: Cache this
1435 flag_name = "invariant.load";
1436 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1437 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
1438 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1444 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
1448 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
1450 MonoCompile *cfg = ctx->cfg;
1452 LLVMBuilderRef builder = *builder_ref;
1455 clause_index = get_handler_clause (cfg, bb);
1457 if (clause_index != -1) {
1458 MonoMethodHeader *header = cfg->header;
1459 MonoExceptionClause *ec = &header->clauses [clause_index];
1460 MonoBasicBlock *tblock;
1461 LLVMBasicBlockRef ex_bb, noex_bb;
1464 * Have to use an invoke instead of a call, branching to the
1465 * handler bblock of the clause containing this bblock.
1468 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1470 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
1473 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
1475 ex_bb = get_bb (ctx, tblock);
1477 noex_bb = gen_bb (ctx, "NOEX_BB");
1480 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1482 builder = ctx->builder = create_builder (ctx);
1483 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1485 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1487 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1488 ctx->builder = builder;
1491 *builder_ref = ctx->builder;
1496 #if LLVM_API_VERSION >= 4
1497 #define EXTRA_MONO_LOAD_STORE_ARGS 1
1499 #define EXTRA_MONO_LOAD_STORE_ARGS 0
1503 emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting, BarrierKind barrier)
1505 const char *intrins_name;
1506 LLVMValueRef args [16], res;
1507 LLVMTypeRef addr_type;
1509 if (is_faulting && bb->region != -1) {
1510 #if LLVM_API_VERSION >= 4
1511 LLVMAtomicOrdering ordering;
1514 case LLVM_BARRIER_NONE:
1515 ordering = LLVMAtomicOrderingNotAtomic;
1517 case LLVM_BARRIER_ACQ:
1518 ordering = LLVMAtomicOrderingAcquire;
1520 case LLVM_BARRIER_SEQ:
1521 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1524 g_assert_not_reached ();
1530 * We handle loads which can fault by calling a mono specific intrinsic
1531 * using an invoke, so they are handled properly inside try blocks.
1532 * We can't use this outside clauses, since LLVM optimizes intrinsics which
1533 * are marked with IntrReadArgMem.
1537 intrins_name = "llvm.mono.load.i8.p0i8";
1540 intrins_name = "llvm.mono.load.i16.p0i16";
1543 intrins_name = "llvm.mono.load.i32.p0i32";
1546 intrins_name = "llvm.mono.load.i64.p0i64";
1549 g_assert_not_reached ();
1552 addr_type = LLVMTypeOf (addr);
1553 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
1554 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1557 args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1558 args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1559 #if LLVM_API_VERSION >= 4
1560 args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1562 res = emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->module, intrins_name), args, 3 + EXTRA_MONO_LOAD_STORE_ARGS);
1564 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
1565 res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
1566 else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
1567 res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
1574 * We emit volatile loads for loads which can fault, because otherwise
1575 * LLVM will generate invalid code when encountering a load from a
1578 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting, barrier);
1580 /* Mark it with a custom metadata */
1583 set_metadata_flag (res, "mono.faulting.load");
1591 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
1593 return emit_load_general (ctx, bb, builder_ref, size, addr, name, is_faulting, LLVM_BARRIER_NONE);
1597 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting, BarrierKind barrier)
1599 const char *intrins_name;
1600 LLVMValueRef args [16];
1602 if (is_faulting && bb->region != -1) {
1603 #if LLVM_API_VERSION >= 4
1604 LLVMAtomicOrdering ordering;
1607 case LLVM_BARRIER_NONE:
1608 ordering = LLVMAtomicOrderingNotAtomic;
1610 case LLVM_BARRIER_REL:
1611 ordering = LLVMAtomicOrderingRelease;
1613 case LLVM_BARRIER_SEQ:
1614 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1617 g_assert_not_reached ();
1624 intrins_name = "llvm.mono.store.i8.p0i8";
1627 intrins_name = "llvm.mono.store.i16.p0i16";
1630 intrins_name = "llvm.mono.store.i32.p0i32";
1633 intrins_name = "llvm.mono.store.i64.p0i64";
1636 g_assert_not_reached ();
1639 if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
1640 value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
1641 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1646 args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1647 args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1648 #if LLVM_API_VERSION >= 4
1649 args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1651 emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->module, intrins_name), args, 4 + EXTRA_MONO_LOAD_STORE_ARGS);
1653 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
1658 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting)
1660 emit_store_general (ctx, bb, builder_ref, size, value, addr, is_faulting, LLVM_BARRIER_NONE);
1664 * emit_cond_system_exception:
1666 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
1667 * Might set the ctx exception.
1670 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
1672 LLVMBasicBlockRef ex_bb, noex_bb;
1673 LLVMBuilderRef builder;
1674 MonoClass *exc_class;
1675 LLVMValueRef args [2];
1677 ex_bb = gen_bb (ctx, "EX_BB");
1678 noex_bb = gen_bb (ctx, "NOEX_BB");
1680 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
1682 exc_class = mono_class_from_name (mono_get_corlib (), "System", exc_type);
1683 g_assert (exc_class);
1685 /* Emit exception throwing code */
1686 builder = create_builder (ctx);
1687 LLVMPositionBuilderAtEnd (builder, ex_bb);
1689 if (!ctx->lmodule->throw_corlib_exception) {
1690 LLVMValueRef callee;
1692 const char *icall_name;
1694 MonoMethodSignature *throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 2);
1695 throw_sig->ret = &mono_get_void_class ()->byval_arg;
1696 throw_sig->params [0] = &mono_get_int32_class ()->byval_arg;
1697 icall_name = "llvm_throw_corlib_exception_abs_trampoline";
1698 /* This will become i8* */
1699 throw_sig->params [1] = &mono_get_byte_class ()->this_arg;
1700 sig = sig_to_llvm_sig (ctx, throw_sig);
1702 if (ctx->cfg->compile_aot) {
1703 callee = get_plt_entry (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
1705 callee = LLVMAddFunction (ctx->module, "llvm_throw_corlib_exception_trampoline", sig_to_llvm_sig (ctx, throw_sig));
1708 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
1709 * - On x86, LLVM generated code doesn't push the arguments
1710 * - The trampoline takes the throw address as an arguments, not a pc offset.
1712 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
1715 mono_memory_barrier ();
1716 ctx->lmodule->throw_corlib_exception = callee;
1719 if (IS_TARGET_X86 || IS_TARGET_AMD64)
1720 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
1722 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE);
1725 * The LLVM mono branch contains changes so a block address can be passed as an
1726 * argument to a call.
1728 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
1729 emit_call (ctx, bb, &builder, ctx->lmodule->throw_corlib_exception, args, 2);
1731 LLVMBuildUnreachable (builder);
1733 ctx->builder = create_builder (ctx);
1734 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1736 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1743 * emit_args_to_vtype:
1745 * Emit code to store the vtype in the arguments args to the address ADDRESS.
1748 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
1750 int j, size, nslots;
1752 size = get_vtype_size (t);
1754 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1755 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
1758 if (ainfo->storage == LLVMArgAsFpArgs)
1759 nslots = ainfo->nslots;
1763 for (j = 0; j < nslots; ++j) {
1764 LLVMValueRef index [2], addr, daddr;
1765 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
1766 LLVMTypeRef part_type;
1768 if (ainfo->pair_storage [j] == LLVMArgNone)
1771 switch (ainfo->pair_storage [j]) {
1772 case LLVMArgInIReg: {
1773 part_type = LLVMIntType (part_size * 8);
1774 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1775 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1776 addr = LLVMBuildGEP (builder, address, index, 1, "");
1778 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
1779 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1780 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1782 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
1785 case LLVMArgInFPReg: {
1786 LLVMTypeRef arg_type;
1788 if (ainfo->esize == 8)
1789 arg_type = LLVMDoubleType ();
1791 arg_type = LLVMFloatType ();
1793 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1794 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
1795 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1796 LLVMBuildStore (builder, args [j], addr);
1802 g_assert_not_reached ();
1805 size -= sizeof (gpointer);
1810 * emit_vtype_to_args:
1812 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
1813 * into ARGS, and the number of arguments into NARGS.
1816 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
1819 int j, size, nslots;
1820 LLVMTypeRef arg_type;
1822 size = get_vtype_size (t);
1824 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t)))
1825 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
1827 if (ainfo->storage == LLVMArgAsFpArgs)
1828 nslots = ainfo->nslots;
1831 for (j = 0; j < nslots; ++j) {
1832 LLVMValueRef index [2], addr, daddr;
1833 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
1835 if (ainfo->pair_storage [j] == LLVMArgNone)
1838 switch (ainfo->pair_storage [j]) {
1840 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1841 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1842 addr = LLVMBuildGEP (builder, address, index, 1, "");
1844 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
1845 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1846 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1848 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
1850 case LLVMArgInFPReg:
1851 if (ainfo->esize == 8)
1852 arg_type = LLVMDoubleType ();
1854 arg_type = LLVMFloatType ();
1855 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
1856 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1857 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1858 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
1863 g_assert_not_reached ();
1865 size -= sizeof (gpointer);
1872 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
1875 * Have to place all alloca's at the end of the entry bb, since otherwise they would
1876 * get executed every time control reaches them.
1878 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
1880 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, "");
1881 return ctx->last_alloca;
1885 build_alloca (EmitContext *ctx, MonoType *t)
1887 MonoClass *k = mono_class_from_mono_type (t);
1890 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
1893 align = mono_class_min_align (k);
1895 /* Sometimes align is not a power of 2 */
1896 while (mono_is_power_of_two (align) == -1)
1899 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
1903 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
1906 mark_as_used (MonoLLVMModule *lmodule, LLVMValueRef global)
1909 lmodule->used = g_ptr_array_sized_new (16);
1910 g_ptr_array_add (lmodule->used, global);
1914 emit_llvm_used (MonoLLVMModule *lmodule)
1916 LLVMModuleRef module = lmodule->module;
1917 LLVMTypeRef used_type;
1918 LLVMValueRef used, *used_elem;
1924 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), lmodule->used->len);
1925 used = LLVMAddGlobal (module, used_type, "llvm.used");
1926 used_elem = g_new0 (LLVMValueRef, lmodule->used->len);
1927 for (i = 0; i < lmodule->used->len; ++i)
1928 used_elem [i] = LLVMConstBitCast (g_ptr_array_index (lmodule->used, i), LLVMPointerType (LLVMInt8Type (), 0));
1929 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, lmodule->used->len));
1930 LLVMSetLinkage (used, LLVMAppendingLinkage);
1931 LLVMSetSection (used, "llvm.metadata");
1937 * Emit code to load/convert arguments.
1940 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
1943 MonoCompile *cfg = ctx->cfg;
1944 MonoMethodSignature *sig = ctx->sig;
1945 LLVMCallInfo *linfo = ctx->linfo;
1948 ctx->alloca_builder = create_builder (ctx);
1951 * Handle indirect/volatile variables by allocating memory for them
1952 * using 'alloca', and storing their address in a temporary.
1954 for (i = 0; i < cfg->num_varinfo; ++i) {
1955 MonoInst *var = cfg->varinfo [i];
1958 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || mini_type_is_vtype (cfg, var->inst_vtype)) {
1959 vtype = type_to_llvm_type (ctx, var->inst_vtype);
1960 CHECK_FAILURE (ctx);
1961 /* Could be already created by an OP_VPHI */
1962 if (!ctx->addresses [var->dreg])
1963 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
1964 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
1968 for (i = 0; i < sig->param_count; ++i) {
1969 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
1970 int reg = cfg->args [i + sig->hasthis]->dreg;
1972 if (ainfo->storage == LLVMArgVtypeInReg || ainfo->storage == LLVMArgAsFpArgs) {
1973 LLVMValueRef args [8];
1976 /* The argument is received as a set of int/fp arguments, store them into the real argument */
1977 memset (args, 0, sizeof (args));
1978 pindex = ctx->pindexes [i];
1979 if (ainfo->storage == LLVMArgVtypeInReg) {
1980 args [0] = LLVMGetParam (ctx->lmethod, pindex);
1981 if (ainfo->pair_storage [1] != LLVMArgNone)
1982 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
1984 g_assert (ainfo->nslots <= 8);
1985 for (j = 0; j < ainfo->nslots; ++j)
1986 args [j] = LLVMGetParam (ctx->lmethod, ctx->pindexes [i] + j);
1988 ctx->addresses [reg] = build_alloca (ctx, sig->params [i]);
1990 emit_args_to_vtype (ctx, builder, sig->params [i], ctx->addresses [reg], ainfo, args);
1992 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) {
1993 /* Treat these as normal values */
1994 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
1996 } else if (ainfo->storage == LLVMArgVtypeByVal) {
1997 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, ctx->pindexes [i]);
1999 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) {
2000 /* Treat these as normal values */
2001 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2003 } else if (ainfo->storage == LLVMArgAsIArgs) {
2004 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, ctx->pindexes [i]);
2006 ctx->addresses [reg] = build_alloca (ctx, sig->params [i]);
2008 /* The argument is received as an array of ints, store it into the real argument */
2009 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
2011 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->params [i])), type_is_unsigned (ctx, sig->params [i]));
2016 emit_volatile_store (ctx, cfg->vret_addr->dreg);
2018 emit_volatile_store (ctx, cfg->args [0]->dreg);
2019 for (i = 0; i < sig->param_count; ++i)
2020 if (!mini_type_is_vtype (cfg, sig->params [i]))
2021 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
2023 if (sig->hasthis && !cfg->rgctx_var && cfg->generic_sharing_context) {
2024 LLVMValueRef this_alloc;
2027 * The exception handling code needs the location where the this argument was
2028 * stored for gshared methods. We create a separate alloca to hold it, and mark it
2029 * with the "mono.this" custom metadata to tell llvm that it needs to save its
2030 * location into the LSDA.
2032 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
2033 /* This volatile store will keep the alloca alive */
2034 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
2036 set_metadata_flag (this_alloc, "mono.this");
2039 if (cfg->rgctx_var) {
2040 LLVMValueRef rgctx_alloc, store;
2043 * We handle the rgctx arg similarly to the this pointer.
2045 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
2046 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
2047 /* This volatile store will keep the alloca alive */
2048 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
2050 set_metadata_flag (rgctx_alloc, "mono.this");
2054 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
2055 * it needs to continue normally, or return back to the exception handling system.
2057 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
2058 if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER))
2059 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
2060 if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER) && bb->in_scount == 0) {
2064 sprintf (name, "finally_ind_bb%d", bb->block_num);
2065 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
2066 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
2068 ctx->bblocks [bb->block_num].finally_ind = val;
2071 * Create a new bblock which CALL_HANDLER can branch to, because branching to the
2072 * LLVM bblock containing the call to llvm.eh.selector causes problems for the
2073 * LLVM optimizer passes.
2075 sprintf (name, "BB_%d_CALL_HANDLER_TARGET", bb->block_num);
2076 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
2084 /* Have to export this for AOT */
2086 mono_personality (void)
2089 g_assert_not_reached ();
2093 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
2095 MonoCompile *cfg = ctx->cfg;
2096 LLVMModuleRef module = ctx->module;
2097 LLVMValueRef *values = ctx->values;
2098 LLVMValueRef *addresses = ctx->addresses;
2099 MonoCallInst *call = (MonoCallInst*)ins;
2100 MonoMethodSignature *sig = call->signature;
2101 LLVMValueRef callee = NULL, lcall;
2103 LLVMCallInfo *cinfo;
2107 LLVMTypeRef llvm_sig;
2109 gboolean virtual, calli;
2110 LLVMBuilderRef builder = *builder_ref;
2113 if (call->signature->call_convention != MONO_CALL_DEFAULT)
2114 LLVM_FAILURE (ctx, "non-default callconv");
2116 cinfo = call->cinfo;
2117 if (call->rgctx_arg_reg)
2118 cinfo->rgctx_arg = TRUE;
2119 if (call->imt_arg_reg)
2120 cinfo->imt_arg = TRUE;
2122 vretaddr = cinfo && cinfo->ret.storage == LLVMArgVtypeRetAddr;
2124 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo, &sinfo);
2125 CHECK_FAILURE (ctx);
2127 virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE || ins->opcode == OP_RCALL_MEMBASE);
2128 calli = (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG || ins->opcode == OP_RCALL_REG);
2130 /* FIXME: Avoid creating duplicate methods */
2132 if (ins->flags & MONO_INST_HAS_METHOD) {
2136 if (cfg->compile_aot) {
2137 callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
2139 LLVM_FAILURE (ctx, "can't encode patch");
2141 callee = LLVMAddFunction (module, "", llvm_sig);
2144 mono_create_jit_trampoline_in_domain (mono_domain_get (),
2146 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, target);
2150 if (call->method && strstr (call->method->klass->name, "AsyncVoidMethodBuilder"))
2151 /* LLVM miscompiles async methods */
2152 LLVM_FAILURE (ctx, "#13734");
2155 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
2161 memset (&ji, 0, sizeof (ji));
2162 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
2163 ji.data.target = info->name;
2165 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
2167 if (cfg->compile_aot) {
2168 callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
2170 LLVM_FAILURE (ctx, "can't encode patch");
2172 callee = LLVMAddFunction (module, "", llvm_sig);
2173 target = (gpointer)mono_icall_get_wrapper (info);
2174 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, target);
2177 if (cfg->compile_aot) {
2179 if (cfg->abs_patches) {
2180 MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr);
2182 callee = get_plt_entry (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
2184 LLVM_FAILURE (ctx, "can't encode patch");
2188 LLVM_FAILURE (ctx, "aot");
2190 callee = LLVMAddFunction (module, "", llvm_sig);
2192 if (cfg->abs_patches) {
2193 MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr);
2196 * FIXME: Some trampolines might have
2197 * their own calling convention on some platforms.
2199 #ifndef TARGET_AMD64
2200 if (abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER || abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER_V4 ||
2201 abs_ji->type == MONO_PATCH_INFO_MONITOR_EXIT || abs_ji->type == MONO_PATCH_INFO_GENERIC_CLASS_INIT)
2202 LLVM_FAILURE (ctx, "trampoline with own cconv");
2204 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE);
2205 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, target);
2209 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, (gpointer)call->fptr);
2215 int size = sizeof (gpointer);
2218 g_assert (ins->inst_offset % size == 0);
2219 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
2221 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
2223 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
2225 if (ins->flags & MONO_INST_HAS_METHOD) {
2230 * Collect and convert arguments
2232 nargs = (sig->param_count * 2) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg;
2233 len = sizeof (LLVMValueRef) * nargs;
2234 args = alloca (len);
2235 memset (args, 0, len);
2236 l = call->out_ireg_args;
2238 if (call->rgctx_arg_reg) {
2239 g_assert (values [call->rgctx_arg_reg]);
2240 g_assert (sinfo.rgctx_arg_pindex < nargs);
2242 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
2243 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
2244 * it using a volatile load.
2247 if (!ctx->imt_rgctx_loc)
2248 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->lmodule->ptr_type, sizeof (gpointer));
2249 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->lmodule->ptr_type), ctx->imt_rgctx_loc);
2250 args [sinfo.rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
2252 args [sinfo.rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->lmodule->ptr_type);
2255 if (call->imt_arg_reg) {
2256 g_assert (values [call->imt_arg_reg]);
2257 g_assert (sinfo.imt_arg_pindex < nargs);
2259 if (!ctx->imt_rgctx_loc)
2260 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->lmodule->ptr_type, sizeof (gpointer));
2261 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->lmodule->ptr_type), ctx->imt_rgctx_loc);
2262 args [sinfo.imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
2264 args [sinfo.imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->lmodule->ptr_type);
2269 if (!addresses [call->inst.dreg])
2270 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2271 g_assert (sinfo.vret_arg_pindex < nargs);
2272 args [sinfo.vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
2275 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2278 LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL;
2282 pindex = sinfo.this_arg_pindex;
2284 pindex = sinfo.pindexes [i - 1];
2286 pindex = sinfo.pindexes [i];
2289 regpair = (guint32)(gssize)(l->data);
2290 reg = regpair & 0xffffff;
2291 args [pindex] = values [reg];
2292 if (ainfo->storage == LLVMArgVtypeInReg || ainfo->storage == LLVMArgAsFpArgs) {
2295 g_assert (addresses [reg]);
2296 emit_vtype_to_args (ctx, builder, sig->params [i - sig->hasthis], addresses [reg], ainfo, args + pindex, &nargs);
2300 // FIXME: Get rid of the VMOVE
2301 } else if (ainfo->storage == LLVMArgVtypeByVal) {
2302 g_assert (addresses [reg]);
2303 args [pindex] = addresses [reg];
2304 } else if (ainfo->storage == LLVMArgAsIArgs) {
2305 g_assert (addresses [reg]);
2306 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
2308 g_assert (args [pindex]);
2309 if (i == 0 && sig->hasthis)
2310 args [pindex] = convert (ctx, args [pindex], ThisType ());
2312 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, sig->params [i - sig->hasthis]));
2318 // FIXME: Align call sites
2324 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
2327 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
2329 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
2330 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
2332 /* The two can't be used together, so use only one LLVM calling conv to pass them */
2333 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
2335 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
2337 if (call->rgctx_arg_reg)
2338 LLVMAddInstrAttribute (lcall, 1 + sinfo.rgctx_arg_pindex, LLVMInRegAttribute);
2339 if (call->imt_arg_reg)
2340 LLVMAddInstrAttribute (lcall, 1 + sinfo.imt_arg_pindex, LLVMInRegAttribute);
2342 /* Add byval attributes if needed */
2343 for (i = 0; i < sig->param_count; ++i) {
2344 LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i + sig->hasthis] : NULL;
2346 if (ainfo && ainfo->storage == LLVMArgVtypeByVal) {
2347 LLVMAddInstrAttribute (lcall, 1 + sinfo.pindexes [i], LLVMByValAttribute);
2352 * Convert the result
2354 if (cinfo && cinfo->ret.storage == LLVMArgVtypeInReg) {
2355 LLVMValueRef regs [2];
2357 if (!addresses [ins->dreg])
2358 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
2360 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
2361 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
2362 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
2364 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
2365 } else if (cinfo && cinfo->ret.storage == LLVMArgVtypeByVal) {
2366 if (!addresses [call->inst.dreg])
2367 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2368 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
2369 } else if (sig->ret->type != MONO_TYPE_VOID && !vretaddr) {
2370 /* If the method returns an unsigned value, need to zext it */
2371 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
2375 if (!addresses [call->inst.dreg])
2376 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2377 g_assert (sinfo.vret_arg_pindex < nargs);
2378 args [sinfo.vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
2381 *builder_ref = ctx->builder;
2383 g_free (sinfo.pindexes);
2391 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
2393 MonoCompile *cfg = ctx->cfg;
2394 MonoMethodSignature *sig = ctx->sig;
2395 LLVMValueRef method = ctx->lmethod;
2396 LLVMValueRef *values = ctx->values;
2397 LLVMValueRef *addresses = ctx->addresses;
2398 LLVMCallInfo *linfo = ctx->linfo;
2399 LLVMModuleRef module = ctx->module;
2400 BBInfo *bblocks = ctx->bblocks;
2402 LLVMBasicBlockRef cbb;
2403 LLVMBuilderRef builder, starting_builder;
2404 gboolean has_terminator;
2406 LLVMValueRef lhs, rhs;
2409 cbb = get_bb (ctx, bb);
2410 builder = create_builder (ctx);
2411 ctx->builder = builder;
2412 LLVMPositionBuilderAtEnd (builder, cbb);
2414 if (bb == cfg->bb_entry)
2415 emit_entry_bb (ctx, builder);
2416 CHECK_FAILURE (ctx);
2418 if (bb->flags & BB_EXCEPTION_HANDLER) {
2420 LLVMValueRef personality;
2421 LLVMBasicBlockRef target_bb;
2423 static gint32 mapping_inited;
2424 static int ti_generator;
2427 LLVMValueRef type_info;
2430 if (!bblocks [bb->block_num].invoke_target) {
2432 * LLVM asserts if llvm.eh.selector is called from a bblock which
2433 * doesn't have an invoke pointing at it.
2434 * Update: LLVM no longer asserts, but some tests in exceptions.exe now fail.
2436 LLVM_FAILURE (ctx, "handler without invokes");
2439 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
2441 if (cfg->compile_aot) {
2442 /* Use a dummy personality function */
2443 personality = LLVMGetNamedFunction (module, "mono_aot_personality");
2444 g_assert (personality);
2446 personality = LLVMGetNamedFunction (module, "mono_personality");
2447 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
2448 LLVMAddGlobalMapping (ctx->lmodule->ee, personality, mono_personality);
2451 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
2453 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
2456 * Create the type info
2458 sprintf (ti_name, "type_info_%d", ti_generator);
2461 if (cfg->compile_aot) {
2462 /* decode_eh_frame () in aot-runtime.c will decode this */
2463 type_info = LLVMAddGlobal (module, LLVMInt32Type (), ti_name);
2464 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
2467 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
2469 LLVMSetLinkage (type_info, LLVMInternalLinkage);
2472 * Enabling this causes llc to crash:
2473 * http://llvm.org/bugs/show_bug.cgi?id=6102
2475 //LLVM_FAILURE (ctx, "aot+clauses");
2477 // test_0_invalid_unbox_arrays () fails
2478 LLVM_FAILURE (ctx, "aot+clauses");
2482 * After the cfg mempool is freed, the type info will point to stale memory,
2483 * but this is not a problem, since we decode it once in exception_cb during
2486 ti = mono_mempool_alloc (cfg->mempool, sizeof (gint32));
2487 *(gint32*)ti = clause_index;
2489 type_info = LLVMAddGlobal (module, i8ptr, ti_name);
2491 LLVMAddGlobalMapping (ctx->lmodule->ee, type_info, ti);
2495 LLVMTypeRef members [2], ret_type;
2496 LLVMValueRef landing_pad;
2498 members [0] = i8ptr;
2499 members [1] = LLVMInt32Type ();
2500 ret_type = LLVMStructType (members, 2, FALSE);
2502 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
2503 LLVMAddClause (landing_pad, type_info);
2505 /* Store the exception into the exvar */
2506 if (bb->in_scount == 1) {
2507 g_assert (bb->in_scount == 1);
2508 exvar = bb->in_stack [0];
2510 // FIXME: This is shared with filter clauses ?
2511 g_assert (!values [exvar->dreg]);
2513 values [exvar->dreg] = LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj");
2514 emit_volatile_store (ctx, exvar->dreg);
2518 /* Start a new bblock which CALL_HANDLER can branch to */
2519 target_bb = bblocks [bb->block_num].call_handler_target_bb;
2521 LLVMBuildBr (builder, target_bb);
2523 ctx->builder = builder = create_builder (ctx);
2524 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
2526 ctx->bblocks [bb->block_num].end_bblock = target_bb;
2530 has_terminator = FALSE;
2531 starting_builder = builder;
2532 for (ins = bb->code; ins; ins = ins->next) {
2533 const char *spec = LLVM_INS_INFO (ins->opcode);
2535 char dname_buf [128];
2537 emit_dbg_loc (ctx, builder, ins->cil_code);
2540 if (nins > 5000 && builder == starting_builder) {
2541 /* some steps in llc are non-linear in the size of basic blocks, see #5714 */
2542 LLVM_FAILURE (ctx, "basic block too long");
2546 /* There could be instructions after a terminator, skip them */
2549 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
2550 sprintf (dname_buf, "t%d", ins->dreg);
2554 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
2555 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
2557 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
2558 lhs = emit_volatile_load (ctx, ins->sreg1);
2560 /* It is ok for SETRET to have an uninitialized argument */
2561 if (!values [ins->sreg1] && ins->opcode != OP_SETRET)
2562 LLVM_FAILURE (ctx, "sreg1");
2563 lhs = values [ins->sreg1];
2569 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
2570 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
2571 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
2572 rhs = emit_volatile_load (ctx, ins->sreg2);
2574 if (!values [ins->sreg2])
2575 LLVM_FAILURE (ctx, "sreg2");
2576 rhs = values [ins->sreg2];
2582 //mono_print_ins (ins);
2583 switch (ins->opcode) {
2586 case OP_LIVERANGE_START:
2587 case OP_LIVERANGE_END:
2590 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
2593 #if SIZEOF_VOID_P == 4
2594 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
2596 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
2600 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
2604 values [ins->dreg] = LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0);
2606 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
2608 case OP_DUMMY_ICONST:
2609 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2611 case OP_DUMMY_I8CONST:
2612 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
2614 case OP_DUMMY_R8CONST:
2615 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
2618 LLVMBuildBr (builder, get_bb (ctx, ins->inst_target_bb));
2619 has_terminator = TRUE;
2625 LLVMBasicBlockRef new_bb;
2626 LLVMBuilderRef new_builder;
2628 // The default branch is already handled
2629 // FIXME: Handle it here
2631 /* Start new bblock */
2632 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
2633 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
2635 lhs = convert (ctx, lhs, LLVMInt32Type ());
2636 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
2637 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
2638 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
2640 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
2643 new_builder = create_builder (ctx);
2644 LLVMPositionBuilderAtEnd (new_builder, new_bb);
2645 LLVMBuildUnreachable (new_builder);
2647 has_terminator = TRUE;
2648 g_assert (!ins->next);
2654 if (linfo->ret.storage == LLVMArgVtypeInReg) {
2655 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
2656 LLVMValueRef part1, retval;
2659 size = get_vtype_size (sig->ret);
2661 g_assert (addresses [ins->sreg1]);
2663 g_assert (linfo->ret.pair_storage [0] == LLVMArgInIReg);
2664 g_assert (linfo->ret.pair_storage [1] == LLVMArgNone);
2666 part1 = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMIntType (size * 8), 0), ""), ""), IntPtrType ());
2668 retval = LLVMBuildInsertValue (builder, LLVMGetUndef (ret_type), part1, 0, "");
2670 LLVMBuildRet (builder, retval);
2674 if (linfo->ret.storage == LLVMArgVtypeByVal) {
2675 LLVMValueRef retval;
2677 g_assert (addresses [ins->sreg1]);
2678 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
2679 LLVMBuildRet (builder, retval);
2683 if (linfo->ret.storage == LLVMArgVtypeRetAddr) {
2684 LLVMBuildRetVoid (builder);
2688 if (!lhs || ctx->is_dead [ins->sreg1]) {
2690 * The method did not set its return value, probably because it
2691 * ends with a throw.
2694 LLVMBuildRetVoid (builder);
2696 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
2698 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
2700 has_terminator = TRUE;
2707 case OP_ICOMPARE_IMM:
2708 case OP_LCOMPARE_IMM:
2709 case OP_COMPARE_IMM: {
2713 if (ins->next->opcode == OP_NOP)
2716 if (ins->next->opcode == OP_BR)
2717 /* The comparison result is not needed */
2720 rel = mono_opcode_to_cond (ins->next->opcode);
2722 if (ins->opcode == OP_ICOMPARE_IMM) {
2723 lhs = convert (ctx, lhs, LLVMInt32Type ());
2724 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
2726 if (ins->opcode == OP_LCOMPARE_IMM) {
2727 lhs = convert (ctx, lhs, LLVMInt64Type ());
2728 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
2730 if (ins->opcode == OP_LCOMPARE) {
2731 lhs = convert (ctx, lhs, LLVMInt64Type ());
2732 rhs = convert (ctx, rhs, LLVMInt64Type ());
2734 if (ins->opcode == OP_ICOMPARE) {
2735 lhs = convert (ctx, lhs, LLVMInt32Type ());
2736 rhs = convert (ctx, rhs, LLVMInt32Type ());
2740 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
2741 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
2742 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
2743 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
2746 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
2747 if (ins->opcode == OP_FCOMPARE) {
2748 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
2749 } else if (ins->opcode == OP_RCOMPARE) {
2750 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
2751 } else if (ins->opcode == OP_COMPARE_IMM) {
2752 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0)
2753 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
2755 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
2756 } else if (ins->opcode == OP_LCOMPARE_IMM) {
2757 if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) {
2758 /* The immediate is encoded in two fields */
2759 guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm);
2760 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), "");
2762 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), "");
2765 else if (ins->opcode == OP_COMPARE) {
2766 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
2767 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
2769 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
2771 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
2773 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
2774 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
2776 * If the target bb contains PHI instructions, LLVM requires
2777 * two PHI entries for this bblock, while we only generate one.
2778 * So convert this to an unconditional bblock. (bxc #171).
2780 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
2782 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
2784 has_terminator = TRUE;
2785 } else if (MONO_IS_SETCC (ins->next)) {
2786 sprintf (dname_buf, "t%d", ins->next->dreg);
2788 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
2790 /* Add stores for volatile variables */
2791 emit_volatile_store (ctx, ins->next->dreg);
2792 } else if (MONO_IS_COND_EXC (ins->next)) {
2793 emit_cond_system_exception (ctx, bb, ins->next->inst_p1, cmp);
2794 CHECK_FAILURE (ctx);
2795 builder = ctx->builder;
2797 LLVM_FAILURE (ctx, "next");
2811 rel = mono_opcode_to_cond (ins->opcode);
2813 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
2814 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
2825 rel = mono_opcode_to_cond (ins->opcode);
2827 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
2828 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
2836 gboolean empty = TRUE;
2838 /* Check that all input bblocks really branch to us */
2839 for (i = 0; i < bb->in_count; ++i) {
2840 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
2841 ins->inst_phi_args [i + 1] = -1;
2847 /* LLVM doesn't like phi instructions with zero operands */
2848 ctx->is_dead [ins->dreg] = TRUE;
2852 /* Created earlier, insert it now */
2853 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
2855 for (i = 0; i < ins->inst_phi_args [0]; i++) {
2856 int sreg1 = ins->inst_phi_args [i + 1];
2860 * Count the number of times the incoming bblock branches to us,
2861 * since llvm requires a separate entry for each.
2863 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
2864 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
2867 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
2868 if (switch_ins->inst_many_bb [j] == bb)
2875 /* Remember for later */
2876 for (j = 0; j < count; ++j) {
2877 PhiNode *node = mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
2880 node->in_bb = bb->in_bb [i];
2882 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
2892 values [ins->dreg] = lhs;
2896 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
2899 values [ins->dreg] = lhs;
2901 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
2903 * This is added by the spilling pass in case of the JIT,
2904 * but we have to do it ourselves.
2906 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
2910 case OP_MOVE_F_TO_I4: {
2911 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
2914 case OP_MOVE_I4_TO_F: {
2915 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
2918 case OP_MOVE_F_TO_I8: {
2919 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
2922 case OP_MOVE_I8_TO_F: {
2923 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
2956 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
2957 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
2959 switch (ins->opcode) {
2962 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
2966 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
2970 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
2974 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
2978 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
2982 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
2986 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
2990 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
2994 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
2998 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
3002 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
3006 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
3010 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
3014 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
3018 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
3021 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
3024 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
3028 g_assert_not_reached ();
3035 lhs = convert (ctx, lhs, LLVMFloatType ());
3036 rhs = convert (ctx, rhs, LLVMFloatType ());
3037 switch (ins->opcode) {
3039 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
3042 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
3045 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
3048 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
3051 g_assert_not_reached ();
3060 case OP_IREM_UN_IMM:
3062 case OP_IDIV_UN_IMM:
3068 case OP_ISHR_UN_IMM:
3077 case OP_LSHR_UN_IMM:
3083 case OP_SHR_UN_IMM: {
3086 if (spec [MONO_INST_SRC1] == 'l') {
3087 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
3089 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
3092 #if SIZEOF_VOID_P == 4
3093 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
3094 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
3097 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
3098 lhs = convert (ctx, lhs, IntPtrType ());
3099 imm = convert (ctx, imm, LLVMTypeOf (lhs));
3100 switch (ins->opcode) {
3104 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
3108 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
3112 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
3116 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
3118 case OP_IDIV_UN_IMM:
3119 case OP_LDIV_UN_IMM:
3120 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
3124 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
3126 case OP_IREM_UN_IMM:
3127 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
3132 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
3136 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
3140 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
3145 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
3150 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
3152 case OP_ISHR_UN_IMM:
3153 /* This is used to implement conv.u4, so the lhs could be an i8 */
3154 lhs = convert (ctx, lhs, LLVMInt32Type ());
3155 imm = convert (ctx, imm, LLVMInt32Type ());
3156 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
3158 case OP_LSHR_UN_IMM:
3160 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
3163 g_assert_not_reached ();
3168 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
3171 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
3174 lhs = convert (ctx, lhs, LLVMDoubleType ());
3175 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
3178 lhs = convert (ctx, lhs, LLVMFloatType ());
3179 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
3182 guint32 v = 0xffffffff;
3183 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
3187 guint64 v = 0xffffffffffffffffLL;
3188 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
3191 #if defined(TARGET_X86) || defined(TARGET_AMD64)
3193 LLVMValueRef v1, v2;
3195 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
3196 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
3197 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
3202 case OP_ICONV_TO_I1:
3203 case OP_ICONV_TO_I2:
3204 case OP_ICONV_TO_I4:
3205 case OP_ICONV_TO_U1:
3206 case OP_ICONV_TO_U2:
3207 case OP_ICONV_TO_U4:
3208 case OP_LCONV_TO_I1:
3209 case OP_LCONV_TO_I2:
3210 case OP_LCONV_TO_U1:
3211 case OP_LCONV_TO_U2:
3212 case OP_LCONV_TO_U4: {
3215 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
3217 /* Have to do two casts since our vregs have type int */
3218 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
3220 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
3222 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
3225 case OP_ICONV_TO_I8:
3226 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
3228 case OP_ICONV_TO_U8:
3229 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
3231 case OP_FCONV_TO_I4:
3232 case OP_RCONV_TO_I4:
3233 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
3235 case OP_FCONV_TO_I1:
3236 case OP_RCONV_TO_I1:
3237 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
3239 case OP_FCONV_TO_U1:
3240 case OP_RCONV_TO_U1:
3241 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
3243 case OP_FCONV_TO_I2:
3244 case OP_RCONV_TO_I2:
3245 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
3247 case OP_FCONV_TO_U2:
3248 case OP_RCONV_TO_U2:
3249 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
3251 case OP_FCONV_TO_I8:
3252 case OP_RCONV_TO_I8:
3253 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
3256 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
3258 case OP_ICONV_TO_R8:
3259 case OP_LCONV_TO_R8:
3260 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
3262 case OP_LCONV_TO_R_UN:
3263 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
3265 #if SIZEOF_VOID_P == 4
3268 case OP_LCONV_TO_I4:
3269 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
3271 case OP_ICONV_TO_R4:
3272 case OP_LCONV_TO_R4:
3273 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
3275 values [ins->dreg] = v;
3277 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
3279 case OP_FCONV_TO_R4:
3280 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
3282 values [ins->dreg] = v;
3284 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
3286 case OP_RCONV_TO_R8:
3287 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
3289 case OP_RCONV_TO_R4:
3290 values [ins->dreg] = lhs;
3293 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
3296 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
3299 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
3301 case OP_LOCALLOC_IMM: {
3304 guint32 size = ins->inst_imm;
3305 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
3307 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
3309 if (ins->flags & MONO_INST_INIT) {
3310 LLVMValueRef args [5];
3313 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3314 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
3315 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
3316 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3317 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, "");
3320 values [ins->dreg] = v;
3324 LLVMValueRef v, size;
3326 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
3328 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
3330 if (ins->flags & MONO_INST_INIT) {
3331 LLVMValueRef args [5];
3334 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3336 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
3337 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3338 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, "");
3340 values [ins->dreg] = v;
3344 case OP_LOADI1_MEMBASE:
3345 case OP_LOADU1_MEMBASE:
3346 case OP_LOADI2_MEMBASE:
3347 case OP_LOADU2_MEMBASE:
3348 case OP_LOADI4_MEMBASE:
3349 case OP_LOADU4_MEMBASE:
3350 case OP_LOADI8_MEMBASE:
3351 case OP_LOADR4_MEMBASE:
3352 case OP_LOADR8_MEMBASE:
3353 case OP_LOAD_MEMBASE:
3361 LLVMValueRef base, index, addr;
3363 gboolean sext = FALSE, zext = FALSE;
3364 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3366 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3371 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
3372 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
3377 if (ins->inst_offset == 0) {
3379 } else if (ins->inst_offset % size != 0) {
3380 /* Unaligned load */
3381 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
3382 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
3384 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3385 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
3389 addr = convert (ctx, addr, LLVMPointerType (t, 0));
3391 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, dname, is_volatile);
3393 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
3395 * These will signal LLVM that these loads do not alias any stores, and
3396 * they can't fail, allowing them to be hoisted out of loops.
3398 set_invariant_load_flag (values [ins->dreg]);
3399 set_metadata_flag (values [ins->dreg], "mono.nofail.load");
3403 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3405 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3406 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
3407 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
3411 case OP_STOREI1_MEMBASE_REG:
3412 case OP_STOREI2_MEMBASE_REG:
3413 case OP_STOREI4_MEMBASE_REG:
3414 case OP_STOREI8_MEMBASE_REG:
3415 case OP_STORER4_MEMBASE_REG:
3416 case OP_STORER8_MEMBASE_REG:
3417 case OP_STORE_MEMBASE_REG: {
3419 LLVMValueRef index, addr;
3421 gboolean sext = FALSE, zext = FALSE;
3422 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3424 if (!values [ins->inst_destbasereg])
3425 LLVM_FAILURE (ctx, "inst_destbasereg");
3427 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3429 if (ins->inst_offset % size != 0) {
3430 /* Unaligned store */
3431 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
3432 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
3434 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3435 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
3437 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
3441 case OP_STOREI1_MEMBASE_IMM:
3442 case OP_STOREI2_MEMBASE_IMM:
3443 case OP_STOREI4_MEMBASE_IMM:
3444 case OP_STOREI8_MEMBASE_IMM:
3445 case OP_STORE_MEMBASE_IMM: {
3447 LLVMValueRef index, addr;
3449 gboolean sext = FALSE, zext = FALSE;
3450 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3452 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3454 if (ins->inst_offset % size != 0) {
3455 /* Unaligned store */
3456 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
3457 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
3459 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3460 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
3462 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
3467 emit_load (ctx, bb, &builder, sizeof (gpointer), convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
3469 case OP_OUTARG_VTRETADDR:
3477 case OP_VOIDCALL_MEMBASE:
3478 case OP_CALL_MEMBASE:
3479 case OP_LCALL_MEMBASE:
3480 case OP_FCALL_MEMBASE:
3481 case OP_RCALL_MEMBASE:
3482 case OP_VCALL_MEMBASE:
3483 case OP_VOIDCALL_REG:
3488 case OP_VCALL_REG: {
3489 process_call (ctx, bb, &builder, ins);
3490 CHECK_FAILURE (ctx);
3495 LLVMValueRef indexes [2];
3497 LLVMValueRef got_entry_addr;
3500 * FIXME: Can't allocate from the cfg mempool since that is freed if
3501 * the LLVM compile fails.
3503 ji = g_new0 (MonoJumpInfo, 1);
3504 ji->type = (MonoJumpInfoType)ins->inst_i1;
3505 ji->data.target = ins->inst_p0;
3507 ji = mono_aot_patch_info_dup (ji);
3509 ji->next = cfg->patch_info;
3510 cfg->patch_info = ji;
3512 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3513 got_offset = mono_aot_get_got_offset (cfg->patch_info);
3514 ctx->lmodule->max_got_offset = MAX (ctx->lmodule->max_got_offset, got_offset);
3516 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3517 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
3518 got_entry_addr = LLVMBuildGEP (builder, ctx->lmodule->got_var, indexes, 2, "");
3520 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, dname);
3521 set_invariant_load_flag (values [ins->dreg]);
3524 case OP_NOT_REACHED:
3525 LLVMBuildUnreachable (builder);
3526 has_terminator = TRUE;
3527 g_assert (bb->block_num < cfg->max_block_num);
3528 ctx->unreachable [bb->block_num] = TRUE;
3529 /* Might have instructions after this */
3531 MonoInst *next = ins->next;
3533 * FIXME: If later code uses the regs defined by these instructions,
3534 * compilation will fail.
3536 MONO_DELETE_INS (bb, next);
3540 MonoInst *var = ins->inst_p0;
3542 values [ins->dreg] = addresses [var->dreg];
3546 LLVMValueRef args [1];
3548 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3549 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sin.f64"), args, 1, dname);
3553 LLVMValueRef args [1];
3555 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3556 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.cos.f64"), args, 1, dname);
3560 LLVMValueRef args [1];
3563 /* This no longer seems to happen */
3565 * LLVM optimizes sqrt(nan) into undefined in
3566 * lib/Analysis/ConstantFolding.cpp
3567 * Also, sqrt(NegativeInfinity) is optimized into 0.
3569 LLVM_FAILURE (ctx, "sqrt");
3571 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3572 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sqrt.f64"), args, 1, dname);
3576 LLVMValueRef args [1];
3578 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3579 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "fabs"), args, 1, dname);
3593 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
3594 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
3596 switch (ins->opcode) {
3599 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
3603 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
3607 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
3611 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
3614 g_assert_not_reached ();
3617 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
3620 case OP_ATOMIC_EXCHANGE_I4:
3621 case OP_ATOMIC_EXCHANGE_I8: {
3622 LLVMValueRef args [2];
3625 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
3626 t = LLVMInt32Type ();
3628 t = LLVMInt64Type ();
3630 g_assert (ins->inst_offset == 0);
3632 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
3633 args [1] = convert (ctx, rhs, t);
3635 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
3638 case OP_ATOMIC_ADD_I4:
3639 case OP_ATOMIC_ADD_I8: {
3640 LLVMValueRef args [2];
3643 if (ins->opcode == OP_ATOMIC_ADD_I4)
3644 t = LLVMInt32Type ();
3646 t = LLVMInt64Type ();
3648 g_assert (ins->inst_offset == 0);
3650 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
3651 args [1] = convert (ctx, rhs, t);
3652 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
3655 case OP_ATOMIC_CAS_I4:
3656 case OP_ATOMIC_CAS_I8: {
3657 LLVMValueRef args [3], val;
3660 if (ins->opcode == OP_ATOMIC_CAS_I4)
3661 t = LLVMInt32Type ();
3663 t = LLVMInt64Type ();
3665 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
3667 args [1] = convert (ctx, values [ins->sreg3], t);
3669 args [2] = convert (ctx, values [ins->sreg2], t);
3670 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
3671 #if LLVM_API_VERSION >= 1
3672 /* cmpxchg returns a pair */
3673 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
3675 values [ins->dreg] = val;
3679 case OP_MEMORY_BARRIER: {
3680 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
3683 case OP_ATOMIC_LOAD_I1:
3684 case OP_ATOMIC_LOAD_I2:
3685 case OP_ATOMIC_LOAD_I4:
3686 case OP_ATOMIC_LOAD_I8:
3687 case OP_ATOMIC_LOAD_U1:
3688 case OP_ATOMIC_LOAD_U2:
3689 case OP_ATOMIC_LOAD_U4:
3690 case OP_ATOMIC_LOAD_U8:
3691 case OP_ATOMIC_LOAD_R4:
3692 case OP_ATOMIC_LOAD_R8: {
3693 #if LLVM_API_VERSION >= 4
3695 gboolean sext, zext;
3697 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3698 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
3699 LLVMValueRef index, addr;
3701 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3706 if (ins->inst_offset != 0) {
3707 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3708 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
3713 addr = convert (ctx, addr, LLVMPointerType (t, 0));
3715 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, dname, is_volatile, barrier);
3718 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3720 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3722 LLVM_FAILURE (ctx, "atomic mono.load intrinsic");
3726 case OP_ATOMIC_STORE_I1:
3727 case OP_ATOMIC_STORE_I2:
3728 case OP_ATOMIC_STORE_I4:
3729 case OP_ATOMIC_STORE_I8:
3730 case OP_ATOMIC_STORE_U1:
3731 case OP_ATOMIC_STORE_U2:
3732 case OP_ATOMIC_STORE_U4:
3733 case OP_ATOMIC_STORE_U8:
3734 case OP_ATOMIC_STORE_R4:
3735 case OP_ATOMIC_STORE_R8: {
3736 #if LLVM_API_VERSION >= 4
3738 gboolean sext, zext;
3740 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3741 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
3742 LLVMValueRef index, addr, value;
3744 if (!values [ins->inst_destbasereg])
3745 LLVM_FAILURE (ctx, "inst_destbasereg");
3747 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3749 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3750 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
3751 value = convert (ctx, values [ins->sreg1], t);
3753 emit_store_general (ctx, bb, &builder, size, value, addr, is_volatile, barrier);
3756 LLVM_FAILURE (ctx, "atomic mono.store intrinsic");
3760 case OP_RELAXED_NOP: {
3761 #if defined(TARGET_AMD64) || defined(TARGET_X86)
3762 emit_call (ctx, bb, &builder, LLVMGetNamedFunction (ctx->module, "llvm.x86.sse2.pause"), NULL, 0);
3769 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
3771 // 257 == FS segment register
3772 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
3774 // 256 == GS segment register
3775 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
3778 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
3779 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
3780 /* See mono_amd64_emit_tls_get () */
3781 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
3783 // 256 == GS segment register
3784 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
3785 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
3787 LLVM_FAILURE (ctx, "opcode tls-get");
3792 case OP_TLS_GET_REG: {
3793 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
3794 /* See emit_tls_get_reg () */
3795 // 256 == GS segment register
3796 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
3797 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt32Type ()), ptrtype, ""), "");
3799 LLVM_FAILURE (ctx, "opcode tls-get");
3808 case OP_IADD_OVF_UN:
3810 case OP_ISUB_OVF_UN:
3812 case OP_IMUL_OVF_UN:
3813 #if SIZEOF_VOID_P == 8
3815 case OP_LADD_OVF_UN:
3817 case OP_LSUB_OVF_UN:
3819 case OP_LMUL_OVF_UN:
3822 LLVMValueRef args [2], val, ovf, func;
3824 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
3825 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
3826 func = LLVMGetNamedFunction (module, ovf_op_to_intrins (ins->opcode));
3828 val = LLVMBuildCall (builder, func, args, 2, "");
3829 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
3830 ovf = LLVMBuildExtractValue (builder, val, 1, "");
3831 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
3832 CHECK_FAILURE (ctx);
3833 builder = ctx->builder;
3839 * We currently model them using arrays. Promotion to local vregs is
3840 * disabled for them in mono_handle_global_vregs () in the LLVM case,
3841 * so we always have an entry in cfg->varinfo for them.
3842 * FIXME: Is this needed ?
3845 MonoClass *klass = ins->klass;
3846 LLVMValueRef args [5];
3850 LLVM_FAILURE (ctx, "!klass");
3854 if (!addresses [ins->dreg])
3855 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3856 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3857 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3858 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
3860 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3861 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3862 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, "");
3865 case OP_DUMMY_VZERO:
3868 case OP_STOREV_MEMBASE:
3869 case OP_LOADV_MEMBASE:
3871 MonoClass *klass = ins->klass;
3872 LLVMValueRef src = NULL, dst, args [5];
3873 gboolean done = FALSE;
3877 LLVM_FAILURE (ctx, "!klass");
3881 if (mini_is_gsharedvt_klass (cfg, klass)) {
3883 LLVM_FAILURE (ctx, "gsharedvt");
3887 switch (ins->opcode) {
3888 case OP_STOREV_MEMBASE:
3889 if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg &&
3890 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
3891 /* Decomposed earlier */
3892 g_assert_not_reached ();
3895 if (!addresses [ins->sreg1]) {
3897 g_assert (values [ins->sreg1]);
3898 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
3899 LLVMBuildStore (builder, values [ins->sreg1], dst);
3902 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
3903 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
3906 case OP_LOADV_MEMBASE:
3907 if (!addresses [ins->dreg])
3908 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3909 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
3910 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3913 if (!addresses [ins->sreg1])
3914 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
3915 if (!addresses [ins->dreg])
3916 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3917 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
3918 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3921 g_assert_not_reached ();
3923 CHECK_FAILURE (ctx);
3930 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
3931 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3933 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3934 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3935 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memcpy_func_name), args, memcpy_param_count, "");
3938 case OP_LLVM_OUTARG_VT:
3939 if (!addresses [ins->sreg1]) {
3940 addresses [ins->sreg1] = build_alloca (ctx, &ins->klass->byval_arg);
3941 g_assert (values [ins->sreg1]);
3942 LLVMBuildStore (builder, values [ins->sreg1], addresses [ins->sreg1]);
3944 addresses [ins->dreg] = addresses [ins->sreg1];
3950 #if defined(TARGET_X86) || defined(TARGET_AMD64)
3952 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
3955 case OP_LOADX_MEMBASE: {
3956 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
3959 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
3960 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
3963 case OP_STOREX_MEMBASE: {
3964 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
3967 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
3968 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
3975 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
3979 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
3985 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
3989 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
3993 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
3997 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
4000 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
4003 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
4006 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
4010 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
4021 LLVMValueRef v = NULL;
4023 switch (ins->opcode) {
4028 t = LLVMVectorType (LLVMInt32Type (), 4);
4029 rt = LLVMVectorType (LLVMFloatType (), 4);
4035 t = LLVMVectorType (LLVMInt64Type (), 2);
4036 rt = LLVMVectorType (LLVMDoubleType (), 2);
4039 t = LLVMInt32Type ();
4040 rt = LLVMInt32Type ();
4041 g_assert_not_reached ();
4044 lhs = LLVMBuildBitCast (builder, lhs, t, "");
4045 rhs = LLVMBuildBitCast (builder, rhs, t, "");
4046 switch (ins->opcode) {
4049 v = LLVMBuildAnd (builder, lhs, rhs, "");
4053 v = LLVMBuildOr (builder, lhs, rhs, "");
4057 v = LLVMBuildXor (builder, lhs, rhs, "");
4061 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
4064 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
4088 case OP_PADDB_SAT_UN:
4089 case OP_PADDW_SAT_UN:
4090 case OP_PSUBB_SAT_UN:
4091 case OP_PSUBW_SAT_UN:
4099 case OP_PMULW_HIGH_UN: {
4100 LLVMValueRef args [2];
4105 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
4112 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, ""), LLVMTypeOf (lhs), "");
4116 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, ""), LLVMTypeOf (lhs), "");
4124 case OP_EXTRACTX_U2:
4126 case OP_EXTRACT_U1: {
4128 gboolean zext = FALSE;
4130 t = simd_op_to_llvm_type (ins->opcode);
4132 switch (ins->opcode) {
4140 case OP_EXTRACTX_U2:
4145 t = LLVMInt32Type ();
4146 g_assert_not_reached ();
4149 lhs = LLVMBuildBitCast (builder, lhs, t, "");
4150 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
4152 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
4161 case OP_EXPAND_R8: {
4162 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
4163 LLVMValueRef mask [16], v;
4166 for (i = 0; i < 16; ++i)
4167 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4169 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
4171 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4172 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
4177 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4180 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4183 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4186 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4189 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4192 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4203 case OP_EXTRACT_MASK:
4210 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
4212 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
4218 LLVMValueRef args [3];
4222 args [2] = LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE);
4224 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 3, dname);
4229 /* This is only used for implementing shifts by non-immediate */
4230 values [ins->dreg] = lhs;
4241 LLVMValueRef args [3];
4244 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4246 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
4257 case OP_PSHLQ_REG: {
4258 LLVMValueRef args [3];
4261 args [1] = values [ins->sreg2];
4263 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
4270 case OP_PSHUFLEW_LOW:
4271 case OP_PSHUFLEW_HIGH: {
4273 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
4274 int i, mask_size = 0;
4275 int imask = ins->inst_c0;
4277 /* Convert the x86 shuffle mask to LLVM's */
4278 switch (ins->opcode) {
4281 mask [0] = ((imask >> 0) & 3);
4282 mask [1] = ((imask >> 2) & 3);
4283 mask [2] = ((imask >> 4) & 3) + 4;
4284 mask [3] = ((imask >> 6) & 3) + 4;
4285 v1 = values [ins->sreg1];
4286 v2 = values [ins->sreg2];
4290 mask [0] = ((imask >> 0) & 1);
4291 mask [1] = ((imask >> 1) & 1) + 2;
4292 v1 = values [ins->sreg1];
4293 v2 = values [ins->sreg2];
4295 case OP_PSHUFLEW_LOW:
4297 mask [0] = ((imask >> 0) & 3);
4298 mask [1] = ((imask >> 2) & 3);
4299 mask [2] = ((imask >> 4) & 3);
4300 mask [3] = ((imask >> 6) & 3);
4305 v1 = values [ins->sreg1];
4306 v2 = LLVMGetUndef (LLVMTypeOf (v1));
4308 case OP_PSHUFLEW_HIGH:
4314 mask [4] = 4 + ((imask >> 0) & 3);
4315 mask [5] = 4 + ((imask >> 2) & 3);
4316 mask [6] = 4 + ((imask >> 4) & 3);
4317 mask [7] = 4 + ((imask >> 6) & 3);
4318 v1 = values [ins->sreg1];
4319 v2 = LLVMGetUndef (LLVMTypeOf (v1));
4323 mask [0] = ((imask >> 0) & 3);
4324 mask [1] = ((imask >> 2) & 3);
4325 mask [2] = ((imask >> 4) & 3);
4326 mask [3] = ((imask >> 6) & 3);
4327 v1 = values [ins->sreg1];
4328 v2 = LLVMGetUndef (LLVMTypeOf (v1));
4331 g_assert_not_reached ();
4333 for (i = 0; i < mask_size; ++i)
4334 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
4336 values [ins->dreg] =
4337 LLVMBuildShuffleVector (builder, v1, v2,
4338 LLVMConstVector (mask_values, mask_size), dname);
4342 case OP_UNPACK_LOWB:
4343 case OP_UNPACK_LOWW:
4344 case OP_UNPACK_LOWD:
4345 case OP_UNPACK_LOWQ:
4346 case OP_UNPACK_LOWPS:
4347 case OP_UNPACK_LOWPD:
4348 case OP_UNPACK_HIGHB:
4349 case OP_UNPACK_HIGHW:
4350 case OP_UNPACK_HIGHD:
4351 case OP_UNPACK_HIGHQ:
4352 case OP_UNPACK_HIGHPS:
4353 case OP_UNPACK_HIGHPD: {
4355 LLVMValueRef mask_values [16];
4356 int i, mask_size = 0;
4357 gboolean low = FALSE;
4359 switch (ins->opcode) {
4360 case OP_UNPACK_LOWB:
4364 case OP_UNPACK_LOWW:
4368 case OP_UNPACK_LOWD:
4369 case OP_UNPACK_LOWPS:
4373 case OP_UNPACK_LOWQ:
4374 case OP_UNPACK_LOWPD:
4378 case OP_UNPACK_HIGHB:
4381 case OP_UNPACK_HIGHW:
4384 case OP_UNPACK_HIGHD:
4385 case OP_UNPACK_HIGHPS:
4388 case OP_UNPACK_HIGHQ:
4389 case OP_UNPACK_HIGHPD:
4393 g_assert_not_reached ();
4397 for (i = 0; i < (mask_size / 2); ++i) {
4399 mask [(i * 2) + 1] = mask_size + i;
4402 for (i = 0; i < (mask_size / 2); ++i) {
4403 mask [(i * 2)] = (mask_size / 2) + i;
4404 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
4408 for (i = 0; i < mask_size; ++i)
4409 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
4411 values [ins->dreg] =
4412 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
4413 LLVMConstVector (mask_values, mask_size), dname);
4418 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
4419 LLVMValueRef v, val;
4421 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4422 val = LLVMConstNull (t);
4423 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4424 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
4426 values [ins->dreg] = val;
4430 case OP_DUPPS_HIGH: {
4431 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
4432 LLVMValueRef v1, v2, val;
4435 if (ins->opcode == OP_DUPPS_LOW) {
4436 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4437 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
4439 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
4440 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
4442 val = LLVMConstNull (t);
4443 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4444 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
4445 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
4446 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
4448 values [ins->dreg] = val;
4458 * EXCEPTION HANDLING
4460 case OP_IMPLICIT_EXCEPTION:
4461 /* This marks a place where an implicit exception can happen */
4462 if (bb->region != -1)
4463 LLVM_FAILURE (ctx, "implicit-exception");
4467 MonoMethodSignature *throw_sig;
4468 LLVMValueRef callee, arg;
4469 gboolean rethrow = (ins->opcode == OP_RETHROW);
4470 const char *icall_name;
4472 callee = rethrow ? ctx->lmodule->rethrow : ctx->lmodule->throw;
4473 icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4476 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4477 throw_sig->ret = &mono_get_void_class ()->byval_arg;
4478 throw_sig->params [0] = &mono_get_object_class ()->byval_arg;
4479 if (cfg->compile_aot) {
4480 callee = get_plt_entry (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
4482 callee = LLVMAddFunction (module, icall_name, sig_to_llvm_sig (ctx, throw_sig));
4486 * LLVM doesn't push the exception argument, so we need a different
4489 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline"));
4491 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
4495 mono_memory_barrier ();
4497 ctx->lmodule->rethrow = callee;
4499 ctx->lmodule->throw = callee;
4501 arg = convert (ctx, lhs, type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg));
4502 emit_call (ctx, bb, &builder, callee, &arg, 1);
4505 case OP_CALL_HANDLER: {
4507 * We don't 'call' handlers, but instead simply branch to them.
4508 * The code generated by ENDFINALLY will branch back to us.
4510 LLVMBasicBlockRef noex_bb;
4512 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
4514 bb_list = info->call_handler_return_bbs;
4517 * Set the indicator variable for the finally clause.
4519 lhs = info->finally_ind;
4521 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
4523 /* Branch to the finally clause */
4524 LLVMBuildBr (builder, info->call_handler_target_bb);
4526 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
4527 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
4529 builder = ctx->builder = create_builder (ctx);
4530 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
4532 bblocks [bb->block_num].end_bblock = noex_bb;
4535 case OP_START_HANDLER: {
4538 case OP_ENDFINALLY: {
4539 LLVMBasicBlockRef resume_bb;
4540 MonoBasicBlock *handler_bb;
4541 LLVMValueRef val, switch_ins, callee;
4545 handler_bb = g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
4546 g_assert (handler_bb);
4547 info = &bblocks [handler_bb->block_num];
4548 lhs = info->finally_ind;
4551 bb_list = info->call_handler_return_bbs;
4553 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
4555 /* Load the finally variable */
4556 val = LLVMBuildLoad (builder, lhs, "");
4558 /* Reset the variable */
4559 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
4561 /* Branch to either resume_bb, or to the bblocks in bb_list */
4562 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
4564 * The other targets are added at the end to handle OP_CALL_HANDLER
4565 * opcodes processed later.
4567 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
4569 builder = ctx->builder = create_builder (ctx);
4570 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
4572 if (ctx->cfg->compile_aot) {
4573 callee = get_plt_entry (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline");
4575 callee = LLVMGetNamedFunction (module, "llvm_resume_unwind_trampoline");
4577 LLVMBuildCall (builder, callee, NULL, 0, "");
4579 LLVMBuildUnreachable (builder);
4580 has_terminator = TRUE;
4586 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
4587 LLVM_FAILURE (ctx, reason);
4592 /* Convert the value to the type required by phi nodes */
4593 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
4594 if (!values [ins->dreg])
4596 values [ins->dreg] = addresses [ins->dreg];
4598 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
4601 /* Add stores for volatile variables */
4602 if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
4603 emit_volatile_store (ctx, ins->dreg);
4606 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0))
4607 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
4609 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
4610 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
4611 LLVMBuildRetVoid (builder);
4614 if (bb == cfg->bb_entry)
4615 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
4624 * mono_llvm_check_method_supported:
4626 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
4627 * compiling a method twice.
4630 mono_llvm_check_method_supported (MonoCompile *cfg)
4632 MonoMethodHeader *header = cfg->header;
4633 MonoExceptionClause *clause;
4636 if (cfg->method->save_lmf) {
4637 cfg->exception_message = g_strdup ("lmf");
4638 cfg->disable_llvm = TRUE;
4640 if (cfg->disable_llvm)
4644 for (i = 0; i < header->num_clauses; ++i) {
4645 clause = &header->clauses [i];
4647 if (i > 0 && clause->try_offset <= header->clauses [i - 1].handler_offset + header->clauses [i - 1].handler_len) {
4649 * FIXME: Some tests still fail with nested clauses.
4651 cfg->exception_message = g_strdup ("nested clauses");
4652 cfg->disable_llvm = TRUE;
4656 if (cfg->disable_llvm)
4661 if (cfg->method->dynamic) {
4662 cfg->exception_message = g_strdup ("dynamic.");
4663 cfg->disable_llvm = TRUE;
4665 if (cfg->disable_llvm)
4670 * mono_llvm_emit_method:
4672 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
4675 mono_llvm_emit_method (MonoCompile *cfg)
4678 MonoMethodSignature *sig;
4680 LLVMTypeRef method_type;
4681 LLVMValueRef method = NULL;
4683 LLVMValueRef *values;
4684 int i, max_block_num, bb_index;
4685 gboolean last = FALSE;
4686 GPtrArray *phi_values;
4687 LLVMCallInfo *linfo;
4689 LLVMModuleRef module;
4691 GPtrArray *bblock_list;
4692 MonoMethodHeader *header;
4693 MonoExceptionClause *clause;
4697 /* The code below might acquire the loader lock, so use it for global locking */
4698 mono_loader_lock ();
4700 /* Used to communicate with the callbacks */
4701 mono_native_tls_set_value (current_cfg_tls_id, cfg);
4703 ctx = g_new0 (EmitContext, 1);
4705 ctx->mempool = cfg->mempool;
4708 * This maps vregs to the LLVM instruction defining them
4710 values = g_new0 (LLVMValueRef, cfg->next_vreg);
4712 * This maps vregs for volatile variables to the LLVM instruction defining their
4715 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
4716 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
4717 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
4718 phi_values = g_ptr_array_sized_new (256);
4720 * This signals whenever the vreg was defined by a phi node with no input vars
4721 * (i.e. all its input bblocks end with NOT_REACHABLE).
4723 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
4724 /* Whenever the bblock is unreachable */
4725 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
4727 bblock_list = g_ptr_array_sized_new (256);
4729 ctx->values = values;
4730 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
4732 if (cfg->compile_aot) {
4733 ctx->lmodule = &aot_module;
4734 method_name = mono_aot_get_method_name (cfg);
4735 cfg->llvm_method_name = g_strdup (method_name);
4737 init_jit_module (cfg->domain);
4738 ctx->lmodule = domain_jit_info (cfg->domain)->llvm_module;
4739 method_name = mono_method_full_name (cfg->method, TRUE);
4742 module = ctx->module = ctx->lmodule->module;
4745 LLVM_FAILURE (ctx, "gsharedvt");
4749 static int count = 0;
4752 if (g_getenv ("LLVM_COUNT")) {
4753 if (count == atoi (g_getenv ("LLVM_COUNT"))) {
4754 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
4758 if (count > atoi (g_getenv ("LLVM_COUNT")))
4759 LLVM_FAILURE (ctx, "");
4764 sig = mono_method_signature (cfg->method);
4767 linfo = mono_arch_get_llvm_call_info (cfg, sig);
4769 CHECK_FAILURE (ctx);
4772 linfo->rgctx_arg = TRUE;
4773 method_type = sig_to_llvm_sig_full (ctx, sig, linfo, &sinfo);
4774 CHECK_FAILURE (ctx);
4777 * This maps parameter indexes in the original signature to the indexes in
4778 * the LLVM signature.
4780 ctx->pindexes = sinfo.pindexes;
4782 method = LLVMAddFunction (module, method_name, method_type);
4783 ctx->lmethod = method;
4785 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
4786 LLVMSetLinkage (method, LLVMPrivateLinkage);
4788 LLVMAddFunctionAttr (method, LLVMUWTable);
4790 if (cfg->compile_aot) {
4791 LLVMSetLinkage (method, LLVMInternalLinkage);
4792 #if LLVM_API_VERSION == 0
4793 /* This causes an assertion in later LLVM versions */
4794 LLVMSetVisibility (method, LLVMHiddenVisibility);
4796 if (ctx->lmodule->external_symbols) {
4797 LLVMSetLinkage (method, LLVMExternalLinkage);
4798 LLVMSetVisibility (method, LLVMHiddenVisibility);
4801 LLVMSetLinkage (method, LLVMPrivateLinkage);
4804 if (cfg->method->save_lmf)
4805 LLVM_FAILURE (ctx, "lmf");
4807 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE)
4808 LLVM_FAILURE (ctx, "pinvoke signature");
4810 header = cfg->header;
4811 for (i = 0; i < header->num_clauses; ++i) {
4812 clause = &header->clauses [i];
4813 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE)
4814 LLVM_FAILURE (ctx, "non-finally/catch clause.");
4817 if (linfo->rgctx_arg) {
4818 ctx->rgctx_arg = LLVMGetParam (method, sinfo.rgctx_arg_pindex);
4820 * We mark the rgctx parameter with the inreg attribute, which is mapped to
4821 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
4822 * CC_X86_64_Mono in X86CallingConv.td.
4824 LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute);
4825 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
4827 if (cfg->vret_addr) {
4828 values [cfg->vret_addr->dreg] = LLVMGetParam (method, sinfo.vret_arg_pindex);
4829 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
4832 values [cfg->args [0]->dreg] = LLVMGetParam (method, sinfo.this_arg_pindex);
4833 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
4836 names = g_new (char *, sig->param_count);
4837 mono_method_get_param_names (cfg->method, (const char **) names);
4839 for (i = 0; i < sig->param_count; ++i) {
4842 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, sinfo.pindexes [i]);
4843 if (names [i] && names [i][0] != '\0')
4844 name = g_strdup_printf ("arg_%s", names [i]);
4846 name = g_strdup_printf ("arg_%d", i);
4847 LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name);
4849 if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeByVal)
4850 LLVMAddAttribute (LLVMGetParam (method, sinfo.pindexes [i]), LLVMByValAttribute);
4854 if (ctx->lmodule->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
4855 ctx->minfo = mono_debug_lookup_method (cfg->method);
4856 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, method_name);
4860 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
4861 max_block_num = MAX (max_block_num, bb->block_num);
4862 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
4864 /* Add branches between non-consecutive bblocks */
4865 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4866 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
4867 bb->next_bb != bb->last_ins->inst_false_bb) {
4869 MonoInst *inst = mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
4870 inst->opcode = OP_BR;
4871 inst->inst_target_bb = bb->last_ins->inst_false_bb;
4872 mono_bblock_add_inst (bb, inst);
4877 * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR
4878 * was later optimized away, so clear these flags, and add them back for the still
4879 * present OP_LDADDR instructions.
4881 for (i = 0; i < cfg->next_vreg; ++i) {
4884 ins = get_vreg_to_inst (cfg, i);
4885 if (ins && ins != cfg->rgctx_var)
4886 ins->flags &= ~MONO_INST_INDIRECT;
4890 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
4892 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4894 LLVMBuilderRef builder;
4896 char dname_buf[128];
4898 builder = create_builder (ctx);
4900 for (ins = bb->code; ins; ins = ins->next) {
4901 switch (ins->opcode) {
4906 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, &ins->klass->byval_arg));
4908 CHECK_FAILURE (ctx);
4910 if (ins->opcode == OP_VPHI) {
4911 /* Treat valuetype PHI nodes as operating on the address itself */
4912 g_assert (ins->klass);
4913 phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
4917 * Have to precreate these, as they can be referenced by
4918 * earlier instructions.
4920 sprintf (dname_buf, "t%d", ins->dreg);
4922 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
4924 if (ins->opcode == OP_VPHI)
4925 ctx->addresses [ins->dreg] = values [ins->dreg];
4927 g_ptr_array_add (phi_values, values [ins->dreg]);
4930 * Set the expected type of the incoming arguments since these have
4931 * to have the same type.
4933 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4934 int sreg1 = ins->inst_phi_args [i + 1];
4937 ctx->vreg_types [sreg1] = phi_type;
4942 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
4951 * Create an ordering for bblocks, use the depth first order first, then
4952 * put the exception handling bblocks last.
4954 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
4955 bb = cfg->bblocks [bb_index];
4956 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
4957 g_ptr_array_add (bblock_list, bb);
4958 bblocks [bb->block_num].added = TRUE;
4962 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4963 if (!bblocks [bb->block_num].added)
4964 g_ptr_array_add (bblock_list, bb);
4968 * Second pass: generate code.
4970 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
4971 bb = g_ptr_array_index (bblock_list, bb_index);
4973 if (!(bb == cfg->bb_entry || bb->in_count > 0))
4976 process_bb (ctx, bb);
4977 CHECK_FAILURE (ctx);
4980 /* Add incoming phi values */
4981 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4982 GSList *l, *ins_list;
4984 ins_list = bblocks [bb->block_num].phi_nodes;
4986 for (l = ins_list; l; l = l->next) {
4987 PhiNode *node = l->data;
4988 MonoInst *phi = node->phi;
4989 int sreg1 = node->sreg;
4990 LLVMBasicBlockRef in_bb;
4995 in_bb = get_end_bb (ctx, node->in_bb);
4997 if (ctx->unreachable [node->in_bb->block_num])
5000 if (!values [sreg1])
5001 /* Can happen with values in EH clauses */
5002 LLVM_FAILURE (ctx, "incoming phi sreg1");
5004 if (phi->opcode == OP_VPHI) {
5005 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
5006 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
5008 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg]))
5010 LLVM_FAILURE (ctx, "incoming phi arg type mismatch");
5011 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
5012 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
5017 /* Create the SWITCH statements for ENDFINALLY instructions */
5018 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5019 BBInfo *info = &bblocks [bb->block_num];
5021 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
5022 LLVMValueRef switch_ins = l->data;
5023 GSList *bb_list = info->call_handler_return_bbs;
5025 for (i = 0; i < g_slist_length (bb_list); ++i)
5026 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data);
5030 if (cfg->verbose_level > 1)
5031 mono_llvm_dump_value (method);
5033 if (cfg->compile_aot)
5034 mark_as_used (ctx->lmodule, method);
5036 if (cfg->compile_aot) {
5037 LLVMValueRef md_args [16];
5038 LLVMValueRef md_node;
5041 method_index = mono_aot_get_method_index (cfg->orig_method);
5042 md_args [0] = LLVMMDString (method_name, strlen (method_name));
5043 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
5044 md_node = LLVMMDNode (md_args, 2);
5045 LLVMAddNamedMetadataOperand (module, "mono.function_indexes", md_node);
5046 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
5049 if (cfg->compile_aot) {
5050 /* Don't generate native code, keep the LLVM IR */
5051 if (cfg->compile_aot && cfg->verbose_level)
5052 printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), method_name);
5054 //LLVMVerifyFunction(method, 0);
5056 mono_llvm_optimize_method (ctx->lmodule->mono_ee, method);
5058 if (cfg->verbose_level > 1)
5059 mono_llvm_dump_value (method);
5061 cfg->native_code = LLVMGetPointerToGlobal (ctx->lmodule->ee, method);
5063 /* Set by emit_cb */
5064 g_assert (cfg->code_len);
5066 /* FIXME: Free the LLVM IL for the function */
5069 if (ctx->lmodule->method_to_lmethod)
5070 g_hash_table_insert (ctx->lmodule->method_to_lmethod, cfg->method, method);
5077 /* Need to add unused phi nodes as they can be referenced by other values */
5078 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (method, "PHI_BB");
5079 LLVMBuilderRef builder;
5081 builder = create_builder (ctx);
5082 LLVMPositionBuilderAtEnd (builder, phi_bb);
5084 for (i = 0; i < phi_values->len; ++i) {
5085 LLVMValueRef v = g_ptr_array_index (phi_values, i);
5086 if (LLVMGetInstructionParent (v) == NULL)
5087 LLVMInsertIntoBuilder (builder, v);
5090 LLVMDeleteFunction (method);
5095 g_free (ctx->addresses);
5096 g_free (ctx->vreg_types);
5097 g_free (ctx->vreg_cli_types);
5098 g_free (ctx->pindexes);
5099 g_free (ctx->is_dead);
5100 g_free (ctx->unreachable);
5101 g_ptr_array_free (phi_values, TRUE);
5102 g_free (ctx->bblocks);
5103 g_hash_table_destroy (ctx->region_to_handler);
5104 g_free (method_name);
5105 g_ptr_array_free (bblock_list, TRUE);
5107 for (l = ctx->builders; l; l = l->next) {
5108 LLVMBuilderRef builder = l->data;
5109 LLVMDisposeBuilder (builder);
5114 mono_native_tls_set_value (current_cfg_tls_id, NULL);
5116 mono_loader_unlock ();
5120 * mono_llvm_emit_call:
5122 * Same as mono_arch_emit_call () for LLVM.
5125 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
5128 MonoMethodSignature *sig;
5129 int i, n, stack_size;
5134 sig = call->signature;
5135 n = sig->param_count + sig->hasthis;
5137 call->cinfo = mono_arch_get_llvm_call_info (cfg, sig);
5139 if (cfg->disable_llvm)
5142 if (sig->call_convention == MONO_CALL_VARARG) {
5143 cfg->exception_message = g_strdup ("varargs");
5144 cfg->disable_llvm = TRUE;
5147 for (i = 0; i < n; ++i) {
5150 ainfo = call->cinfo->args + i;
5152 in = call->args [i];
5154 /* Simply remember the arguments */
5155 switch (ainfo->storage) {
5157 case LLVMArgInFPReg: {
5158 MonoType *t = (sig->hasthis && i == 0) ? &mono_get_intptr_class ()->byval_arg : sig->params [i - sig->hasthis];
5161 opcode = mono_type_to_regmove (cfg, t);
5162 if (opcode == OP_FMOVE) {
5163 MONO_INST_NEW (cfg, ins, OP_FMOVE);
5164 ins->dreg = mono_alloc_freg (cfg);
5165 } else if (opcode == OP_LMOVE) {
5166 MONO_INST_NEW (cfg, ins, OP_LMOVE);
5167 ins->dreg = mono_alloc_lreg (cfg);
5169 MONO_INST_NEW (cfg, ins, OP_MOVE);
5170 ins->dreg = mono_alloc_ireg (cfg);
5172 ins->sreg1 = in->dreg;
5175 case LLVMArgVtypeByVal:
5176 case LLVMArgVtypeInReg:
5177 case LLVMArgAsIArgs:
5178 case LLVMArgAsFpArgs:
5179 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
5180 ins->dreg = mono_alloc_ireg (cfg);
5181 ins->sreg1 = in->dreg;
5182 ins->klass = mono_class_from_mono_type (sig->params [i - sig->hasthis]);
5185 call->cinfo = mono_arch_get_llvm_call_info (cfg, sig);
5186 cfg->exception_message = g_strdup ("ainfo->storage");
5187 cfg->disable_llvm = TRUE;
5191 if (!cfg->disable_llvm) {
5192 MONO_ADD_INS (cfg->cbb, ins);
5193 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
5198 static unsigned char*
5199 alloc_cb (LLVMValueRef function, int size)
5203 cfg = mono_native_tls_get_value (current_cfg_tls_id);
5207 return mono_domain_code_reserve (cfg->domain, size);
5209 return mono_domain_code_reserve (mono_domain_get (), size);
5214 emitted_cb (LLVMValueRef function, void *start, void *end)
5218 cfg = mono_native_tls_get_value (current_cfg_tls_id);
5220 cfg->code_len = (guint8*)end - (guint8*)start;
5224 exception_cb (void *data)
5227 MonoJitExceptionInfo *ei;
5228 guint32 ei_len, i, j, nested_len, nindex;
5229 gpointer *type_info;
5230 int this_reg, this_offset;
5232 cfg = mono_native_tls_get_value (current_cfg_tls_id);
5236 * data points to a DWARF FDE structure, convert it to our unwind format and
5238 * An alternative would be to save it directly, and modify our unwinder to work
5241 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
5242 if (cfg->verbose_level > 1)
5243 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
5245 /* Count nested clauses */
5247 for (i = 0; i < ei_len; ++i) {
5248 for (j = 0; j < ei_len; ++j) {
5249 gint32 cindex1 = *(gint32*)type_info [i];
5250 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
5251 gint32 cindex2 = *(gint32*)type_info [j];
5252 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
5254 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
5260 cfg->llvm_ex_info = mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
5261 cfg->llvm_ex_info_len = ei_len + nested_len;
5262 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
5263 /* Fill the rest of the information from the type info */
5264 for (i = 0; i < ei_len; ++i) {
5265 gint32 clause_index = *(gint32*)type_info [i];
5266 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
5268 cfg->llvm_ex_info [i].flags = clause->flags;
5269 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
5273 * For nested clauses, the LLVM produced exception info associates the try interval with
5274 * the innermost handler, while mono expects it to be associated with all nesting clauses.
5276 /* FIXME: These should be order with the normal clauses */
5278 for (i = 0; i < ei_len; ++i) {
5279 for (j = 0; j < ei_len; ++j) {
5280 gint32 cindex1 = *(gint32*)type_info [i];
5281 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
5282 gint32 cindex2 = *(gint32*)type_info [j];
5283 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
5285 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
5287 * The try interval comes from the nested clause, everything else from the
5290 memcpy (&cfg->llvm_ex_info [nindex], &cfg->llvm_ex_info [j], sizeof (MonoJitExceptionInfo));
5291 cfg->llvm_ex_info [nindex].try_start = cfg->llvm_ex_info [i].try_start;
5292 cfg->llvm_ex_info [nindex].try_end = cfg->llvm_ex_info [i].try_end;
5297 g_assert (nindex == ei_len + nested_len);
5298 cfg->llvm_this_reg = this_reg;
5299 cfg->llvm_this_offset = this_offset;
5301 /* type_info [i] is cfg mempool allocated, no need to free it */
5308 dlsym_cb (const char *name, void **symbol)
5314 if (!strcmp (name, "__bzero")) {
5315 *symbol = (void*)bzero;
5317 current = mono_dl_open (NULL, 0, NULL);
5320 err = mono_dl_symbol (current, name, symbol);
5322 mono_dl_close (current);
5324 #ifdef MONO_ARCH_HAVE_CREATE_LLVM_NATIVE_THUNK
5325 *symbol = (char*)mono_arch_create_llvm_native_thunk (mono_domain_get (), (guint8*)(*symbol));
5331 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
5333 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
5337 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
5339 LLVMTypeRef param_types [4];
5341 param_types [0] = param_type1;
5342 param_types [1] = param_type2;
5344 AddFunc (module, name, ret_type, param_types, 2);
5348 add_intrinsics (LLVMModuleRef module)
5350 /* Emit declarations of instrinsics */
5352 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
5353 * type doesn't seem to do any locking.
5356 LLVMTypeRef memset_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
5358 memset_param_count = 5;
5359 memset_func_name = "llvm.memset.p0i8.i32";
5361 LLVMAddFunction (module, memset_func_name, LLVMFunctionType (LLVMVoidType (), memset_params, memset_param_count, FALSE));
5365 LLVMTypeRef memcpy_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
5367 memcpy_param_count = 5;
5368 memcpy_func_name = "llvm.memcpy.p0i8.p0i8.i32";
5370 LLVMAddFunction (module, memcpy_func_name, LLVMFunctionType (LLVMVoidType (), memcpy_params, memcpy_param_count, FALSE));
5374 LLVMTypeRef params [] = { LLVMDoubleType () };
5376 LLVMAddFunction (module, "llvm.sin.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5377 LLVMAddFunction (module, "llvm.cos.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5378 LLVMAddFunction (module, "llvm.sqrt.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5380 /* This isn't an intrinsic, instead llvm seems to special case it by name */
5381 LLVMAddFunction (module, "fabs", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5385 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
5386 LLVMTypeRef ovf_params_i32 [] = { LLVMInt32Type (), LLVMInt32Type () };
5388 LLVMAddFunction (module, "llvm.sadd.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5389 LLVMAddFunction (module, "llvm.uadd.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5390 LLVMAddFunction (module, "llvm.ssub.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5391 LLVMAddFunction (module, "llvm.usub.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5392 LLVMAddFunction (module, "llvm.smul.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5393 LLVMAddFunction (module, "llvm.umul.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5397 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
5398 LLVMTypeRef ovf_params_i64 [] = { LLVMInt64Type (), LLVMInt64Type () };
5400 LLVMAddFunction (module, "llvm.sadd.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5401 LLVMAddFunction (module, "llvm.uadd.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5402 LLVMAddFunction (module, "llvm.ssub.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5403 LLVMAddFunction (module, "llvm.usub.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5404 LLVMAddFunction (module, "llvm.smul.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5405 LLVMAddFunction (module, "llvm.umul.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5410 LLVMTypeRef arg_types [2];
5411 LLVMTypeRef ret_type;
5413 arg_types [0] = LLVMPointerType (LLVMInt8Type (), 0);
5414 arg_types [1] = LLVMPointerType (LLVMInt8Type (), 0);
5415 ret_type = LLVMInt32Type ();
5417 LLVMAddFunction (module, "mono_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
5419 LLVMAddFunction (module, "llvm_resume_unwind_trampoline", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
5422 /* SSE intrinsics */
5423 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5425 LLVMTypeRef ret_type, arg_types [16];
5428 ret_type = type_to_simd_type (MONO_TYPE_I4);
5429 arg_types [0] = ret_type;
5430 arg_types [1] = ret_type;
5431 AddFunc (module, "llvm.x86.sse41.pminud", ret_type, arg_types, 2);
5432 AddFunc (module, "llvm.x86.sse41.pmaxud", ret_type, arg_types, 2);
5434 ret_type = type_to_simd_type (MONO_TYPE_I2);
5435 arg_types [0] = ret_type;
5436 arg_types [1] = ret_type;
5437 AddFunc (module, "llvm.x86.sse41.pminuw", ret_type, arg_types, 2);
5438 AddFunc (module, "llvm.x86.sse2.pmins.w", ret_type, arg_types, 2);
5439 AddFunc (module, "llvm.x86.sse41.pmaxuw", ret_type, arg_types, 2);
5440 AddFunc (module, "llvm.x86.sse2.padds.w", ret_type, arg_types, 2);
5441 AddFunc (module, "llvm.x86.sse2.psubs.w", ret_type, arg_types, 2);
5442 AddFunc (module, "llvm.x86.sse2.paddus.w", ret_type, arg_types, 2);
5443 AddFunc (module, "llvm.x86.sse2.psubus.w", ret_type, arg_types, 2);
5444 AddFunc (module, "llvm.x86.sse2.pavg.w", ret_type, arg_types, 2);
5445 AddFunc (module, "llvm.x86.sse2.pmulh.w", ret_type, arg_types, 2);
5446 AddFunc (module, "llvm.x86.sse2.pmulhu.w", ret_type, arg_types, 2);
5448 ret_type = type_to_simd_type (MONO_TYPE_I1);
5449 arg_types [0] = ret_type;
5450 arg_types [1] = ret_type;
5451 AddFunc (module, "llvm.x86.sse2.pminu.b", ret_type, arg_types, 2);
5452 AddFunc (module, "llvm.x86.sse2.pmaxu.b", ret_type, arg_types, 2);
5453 AddFunc (module, "llvm.x86.sse2.padds.b", ret_type, arg_types, 2);
5454 AddFunc (module, "llvm.x86.sse2.psubs.b", ret_type, arg_types, 2);
5455 AddFunc (module, "llvm.x86.sse2.paddus.b", ret_type, arg_types, 2);
5456 AddFunc (module, "llvm.x86.sse2.psubus.b", ret_type, arg_types, 2);
5457 AddFunc (module, "llvm.x86.sse2.pavg.b", ret_type, arg_types, 2);
5459 ret_type = type_to_simd_type (MONO_TYPE_R8);
5460 arg_types [0] = ret_type;
5461 arg_types [1] = ret_type;
5462 AddFunc (module, "llvm.x86.sse2.min.pd", ret_type, arg_types, 2);
5463 AddFunc (module, "llvm.x86.sse2.max.pd", ret_type, arg_types, 2);
5464 AddFunc (module, "llvm.x86.sse3.hadd.pd", ret_type, arg_types, 2);
5465 AddFunc (module, "llvm.x86.sse3.hsub.pd", ret_type, arg_types, 2);
5466 AddFunc (module, "llvm.x86.sse3.addsub.pd", ret_type, arg_types, 2);
5468 ret_type = type_to_simd_type (MONO_TYPE_R4);
5469 arg_types [0] = ret_type;
5470 arg_types [1] = ret_type;
5471 AddFunc (module, "llvm.x86.sse.min.ps", ret_type, arg_types, 2);
5472 AddFunc (module, "llvm.x86.sse.max.ps", ret_type, arg_types, 2);
5473 AddFunc (module, "llvm.x86.sse3.hadd.ps", ret_type, arg_types, 2);
5474 AddFunc (module, "llvm.x86.sse3.hsub.ps", ret_type, arg_types, 2);
5475 AddFunc (module, "llvm.x86.sse3.addsub.ps", ret_type, arg_types, 2);
5478 ret_type = type_to_simd_type (MONO_TYPE_I1);
5479 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
5480 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
5481 AddFunc (module, "llvm.x86.sse2.packsswb.128", ret_type, arg_types, 2);
5482 AddFunc (module, "llvm.x86.sse2.packuswb.128", ret_type, arg_types, 2);
5483 ret_type = type_to_simd_type (MONO_TYPE_I2);
5484 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
5485 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
5486 AddFunc (module, "llvm.x86.sse2.packssdw.128", ret_type, arg_types, 2);
5487 AddFunc (module, "llvm.x86.sse41.packusdw", ret_type, arg_types, 2);
5490 ret_type = type_to_simd_type (MONO_TYPE_R8);
5491 arg_types [0] = ret_type;
5492 arg_types [1] = ret_type;
5493 arg_types [2] = LLVMInt8Type ();
5494 AddFunc (module, "llvm.x86.sse2.cmp.pd", ret_type, arg_types, 3);
5495 ret_type = type_to_simd_type (MONO_TYPE_R4);
5496 arg_types [0] = ret_type;
5497 arg_types [1] = ret_type;
5498 arg_types [2] = LLVMInt8Type ();
5499 AddFunc (module, "llvm.x86.sse.cmp.ps", ret_type, arg_types, 3);
5501 /* Conversion ops */
5502 ret_type = type_to_simd_type (MONO_TYPE_R8);
5503 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
5504 AddFunc (module, "llvm.x86.sse2.cvtdq2pd", ret_type, arg_types, 1);
5505 ret_type = type_to_simd_type (MONO_TYPE_R4);
5506 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
5507 AddFunc (module, "llvm.x86.sse2.cvtdq2ps", ret_type, arg_types, 1);
5508 ret_type = type_to_simd_type (MONO_TYPE_I4);
5509 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
5510 AddFunc (module, "llvm.x86.sse2.cvtpd2dq", ret_type, arg_types, 1);
5511 ret_type = type_to_simd_type (MONO_TYPE_I4);
5512 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
5513 AddFunc (module, "llvm.x86.sse2.cvtps2dq", ret_type, arg_types, 1);
5514 ret_type = type_to_simd_type (MONO_TYPE_R4);
5515 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
5516 AddFunc (module, "llvm.x86.sse2.cvtpd2ps", ret_type, arg_types, 1);
5517 ret_type = type_to_simd_type (MONO_TYPE_R8);
5518 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
5519 AddFunc (module, "llvm.x86.sse2.cvtps2pd", ret_type, arg_types, 1);
5521 ret_type = type_to_simd_type (MONO_TYPE_I4);
5522 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
5523 AddFunc (module, "llvm.x86.sse2.cvttpd2dq", ret_type, arg_types, 1);
5524 ret_type = type_to_simd_type (MONO_TYPE_I4);
5525 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
5526 AddFunc (module, "llvm.x86.sse2.cvttps2dq", ret_type, arg_types, 1);
5529 ret_type = type_to_simd_type (MONO_TYPE_R8);
5530 arg_types [0] = ret_type;
5531 AddFunc (module, "llvm.x86.sse2.sqrt.pd", ret_type, arg_types, 1);
5532 ret_type = type_to_simd_type (MONO_TYPE_R4);
5533 arg_types [0] = ret_type;
5534 AddFunc (module, "llvm.x86.sse.sqrt.ps", ret_type, arg_types, 1);
5535 ret_type = type_to_simd_type (MONO_TYPE_R4);
5536 arg_types [0] = ret_type;
5537 AddFunc (module, "llvm.x86.sse.rsqrt.ps", ret_type, arg_types, 1);
5538 ret_type = type_to_simd_type (MONO_TYPE_R4);
5539 arg_types [0] = ret_type;
5540 AddFunc (module, "llvm.x86.sse.rcp.ps", ret_type, arg_types, 1);
5543 ret_type = type_to_simd_type (MONO_TYPE_I2);
5544 arg_types [0] = ret_type;
5545 arg_types [1] = LLVMInt32Type ();
5546 AddFunc (module, "llvm.x86.sse2.psrli.w", ret_type, arg_types, 2);
5547 AddFunc (module, "llvm.x86.sse2.psrai.w", ret_type, arg_types, 2);
5548 AddFunc (module, "llvm.x86.sse2.pslli.w", ret_type, arg_types, 2);
5549 ret_type = type_to_simd_type (MONO_TYPE_I4);
5550 arg_types [0] = ret_type;
5551 arg_types [1] = LLVMInt32Type ();
5552 AddFunc (module, "llvm.x86.sse2.psrli.d", ret_type, arg_types, 2);
5553 AddFunc (module, "llvm.x86.sse2.psrai.d", ret_type, arg_types, 2);
5554 AddFunc (module, "llvm.x86.sse2.pslli.d", ret_type, arg_types, 2);
5555 ret_type = type_to_simd_type (MONO_TYPE_I8);
5556 arg_types [0] = ret_type;
5557 arg_types [1] = LLVMInt32Type ();
5558 AddFunc (module, "llvm.x86.sse2.psrli.q", ret_type, arg_types, 2);
5559 AddFunc (module, "llvm.x86.sse2.pslli.q", ret_type, arg_types, 2);
5562 ret_type = LLVMInt32Type ();
5563 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
5564 AddFunc (module, "llvm.x86.sse2.pmovmskb.128", ret_type, arg_types, 1);
5567 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
5570 /* Load/Store intrinsics */
5572 LLVMTypeRef arg_types [5];
5576 for (i = 1; i <= 8; i *= 2) {
5577 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
5578 arg_types [1] = LLVMInt32Type ();
5579 arg_types [2] = LLVMInt1Type ();
5580 #if LLVM_API_VERSION >= 4
5581 arg_types [3] = LLVMInt32Type ();
5583 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
5584 LLVMAddFunction (module, name, LLVMFunctionType (LLVMIntType (i * 8), arg_types, 3 + EXTRA_MONO_LOAD_STORE_ARGS, FALSE));
5586 arg_types [0] = LLVMIntType (i * 8);
5587 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
5588 arg_types [2] = LLVMInt32Type ();
5589 arg_types [3] = LLVMInt1Type ();
5590 #if LLVM_API_VERSION >= 4
5591 arg_types [4] = LLVMInt32Type ();
5593 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
5594 LLVMAddFunction (module, name, LLVMFunctionType (LLVMVoidType (), arg_types, 4 + EXTRA_MONO_LOAD_STORE_ARGS, FALSE));
5600 add_types (MonoLLVMModule *lmodule)
5602 lmodule->ptr_type = LLVMPointerType (sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
5606 mono_llvm_init (void)
5608 mono_native_tls_alloc (¤t_cfg_tls_id, NULL);
5612 init_jit_module (MonoDomain *domain)
5614 MonoJitICallInfo *info;
5615 MonoJitDomainInfo *dinfo;
5616 MonoLLVMModule *module;
5619 dinfo = domain_jit_info (domain);
5620 if (dinfo->llvm_module)
5623 mono_loader_lock ();
5625 if (dinfo->llvm_module) {
5626 mono_loader_unlock ();
5630 module = g_new0 (MonoLLVMModule, 1);
5632 name = g_strdup_printf ("mono-%s", domain->friendly_name);
5633 module->module = LLVMModuleCreateWithName (name);
5635 module->mono_ee = mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->module), alloc_cb, emitted_cb, exception_cb, dlsym_cb, &module->ee);
5637 add_intrinsics (module->module);
5640 module->llvm_types = g_hash_table_new (NULL, NULL);
5642 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
5644 LLVMAddGlobalMapping (module->ee, LLVMGetNamedFunction (module->module, "llvm_resume_unwind_trampoline"), (void*)info->func);
5646 mono_memory_barrier ();
5648 dinfo->llvm_module = module;
5650 mono_loader_unlock ();
5654 mono_llvm_cleanup (void)
5656 if (aot_module.module)
5657 LLVMDisposeModule (aot_module.module);
5659 LLVMContextDispose (LLVMGetGlobalContext ());
5663 mono_llvm_free_domain_info (MonoDomain *domain)
5665 MonoJitDomainInfo *info = domain_jit_info (domain);
5666 MonoLLVMModule *module = info->llvm_module;
5672 if (module->llvm_types)
5673 g_hash_table_destroy (module->llvm_types);
5675 mono_llvm_dispose_ee (module->mono_ee);
5677 if (module->bb_names) {
5678 for (i = 0; i < module->bb_names_len; ++i)
5679 g_free (module->bb_names [i]);
5680 g_free (module->bb_names);
5682 //LLVMDisposeModule (module->module);
5686 info->llvm_module = NULL;
5690 mono_llvm_create_aot_module (const char *got_symbol, gboolean external_symbols, gboolean emit_dwarf)
5692 /* Delete previous module */
5693 if (aot_module.plt_entries)
5694 g_hash_table_destroy (aot_module.plt_entries);
5695 if (aot_module.module)
5696 LLVMDisposeModule (aot_module.module);
5698 memset (&aot_module, 0, sizeof (aot_module));
5700 aot_module.module = LLVMModuleCreateWithName ("aot");
5701 aot_module.got_symbol = got_symbol;
5702 aot_module.external_symbols = external_symbols;
5703 aot_module.emit_dwarf = emit_dwarf;
5704 /* The first few entries are reserved */
5705 aot_module.max_got_offset = 16;
5707 add_intrinsics (aot_module.module);
5708 add_types (&aot_module);
5712 * We couldn't compute the type of the LLVM global representing the got because
5713 * its size is only known after all the methods have been emitted. So create
5714 * a dummy variable, and replace all uses it with the real got variable when
5715 * its size is known in mono_llvm_emit_aot_module ().
5718 LLVMTypeRef got_type = LLVMArrayType (aot_module.ptr_type, 0);
5720 aot_module.got_var = LLVMAddGlobal (aot_module.module, got_type, "mono_dummy_got");
5721 LLVMSetInitializer (aot_module.got_var, LLVMConstNull (got_type));
5724 /* Add a dummy personality function */
5726 LLVMBasicBlockRef lbb;
5727 LLVMBuilderRef lbuilder;
5728 LLVMValueRef personality;
5730 personality = LLVMAddFunction (aot_module.module, "mono_aot_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
5731 LLVMSetLinkage (personality, LLVMInternalLinkage);
5732 lbb = LLVMAppendBasicBlock (personality, "BB0");
5733 lbuilder = LLVMCreateBuilder ();
5734 LLVMPositionBuilderAtEnd (lbuilder, lbb);
5735 LLVMBuildRetVoid (lbuilder);
5738 aot_module.llvm_types = g_hash_table_new (NULL, NULL);
5739 aot_module.plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
5740 aot_module.plt_entries_ji = g_hash_table_new (NULL, NULL);
5741 aot_module.method_to_lmethod = g_hash_table_new (NULL, NULL);
5745 * Emit the aot module into the LLVM bitcode file FILENAME.
5748 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
5750 LLVMTypeRef got_type;
5751 LLVMValueRef real_got;
5752 MonoLLVMModule *module = &aot_module;
5755 * Create the real got variable and replace all uses of the dummy variable with
5758 got_type = LLVMArrayType (aot_module.ptr_type, module->max_got_offset + 1);
5759 real_got = LLVMAddGlobal (aot_module.module, got_type, aot_module.got_symbol);
5760 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
5761 if (module->external_symbols) {
5762 LLVMSetLinkage (real_got, LLVMExternalLinkage);
5763 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
5765 LLVMSetLinkage (real_got, LLVMInternalLinkage);
5767 mono_llvm_replace_uses_of (aot_module.got_var, real_got);
5769 mark_as_used (&aot_module, real_got);
5771 /* Delete the dummy got so it doesn't become a global */
5772 LLVMDeleteGlobal (aot_module.got_var);
5774 emit_llvm_used (&aot_module);
5775 emit_dbg_info (&aot_module, filename, cu_name);
5777 /* Replace PLT entries for directly callable methods with the methods themselves */
5779 GHashTableIter iter;
5781 LLVMValueRef callee;
5783 g_hash_table_iter_init (&iter, aot_module.plt_entries_ji);
5784 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
5785 if (mono_aot_is_direct_callable (ji)) {
5786 LLVMValueRef lmethod;
5788 lmethod = g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
5789 /* The types might not match because the caller might pass an rgctx */
5790 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
5791 mono_llvm_replace_uses_of (callee, lmethod);
5792 mono_aot_mark_unused_llvm_plt_entry (ji);
5802 if (LLVMVerifyModule (aot_module.module, LLVMReturnStatusAction, &verifier_err)) {
5803 g_assert_not_reached ();
5808 LLVMWriteBitcodeToFile (aot_module.module, filename);
5813 md_string (const char *s)
5815 return LLVMMDString (s, strlen (s));
5818 /* Debugging support */
5821 emit_dbg_info (MonoLLVMModule *lmodule, const char *filename, const char *cu_name)
5823 LLVMModuleRef module = lmodule->module;
5824 LLVMValueRef args [16], cu_args [16], cu, ver;
5826 char *build_info, *s, *dir;
5829 * This can only be enabled when LLVM code is emitted into a separate object
5830 * file, since the AOT compiler also emits dwarf info,
5831 * and the abbrev indexes will not be correct since llvm has added its own
5834 if (!lmodule->emit_dwarf)
5838 * Emit dwarf info in the form of LLVM metadata. There is some
5839 * out-of-date documentation at:
5840 * http://llvm.org/docs/SourceLevelDebugging.html
5841 * but most of this was gathered from the llvm and
5846 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_TAG_compile_unit, FALSE);
5847 /* CU name/compilation dir */
5848 dir = g_path_get_dirname (filename);
5849 args [0] = LLVMMDString (cu_name, strlen (cu_name));
5850 args [1] = LLVMMDString (dir, strlen (dir));
5851 cu_args [n_cuargs ++] = LLVMMDNode (args, 2);
5854 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_LANG_C99, FALSE);
5856 build_info = mono_get_runtime_build_info ();
5857 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
5858 cu_args [n_cuargs ++] = LLVMMDString (s, strlen (s));
5859 g_free (build_info);
5861 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5863 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
5864 /* Runtime version */
5865 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5867 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5868 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5870 if (lmodule->subprogram_mds) {
5874 mds = g_new0 (LLVMValueRef, lmodule->subprogram_mds->len);
5875 for (i = 0; i < lmodule->subprogram_mds->len; ++i)
5876 mds [i] = g_ptr_array_index (lmodule->subprogram_mds, i);
5877 cu_args [n_cuargs ++] = LLVMMDNode (mds, lmodule->subprogram_mds->len);
5879 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5882 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5883 /* Imported modules */
5884 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5886 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
5887 /* DebugEmissionKind = FullDebug */
5888 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5889 cu = LLVMMDNode (cu_args, n_cuargs);
5890 LLVMAddNamedMetadataOperand (module, "llvm.dbg.cu", cu);
5892 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5893 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
5894 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
5895 ver = LLVMMDNode (args, 3);
5896 LLVMAddNamedMetadataOperand (module, "llvm.module.flags", ver);
5898 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5899 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
5900 args [2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5901 ver = LLVMMDNode (args, 3);
5902 LLVMAddNamedMetadataOperand (module, "llvm.module.flags", ver);
5906 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
5908 MonoLLVMModule *module = ctx->lmodule;
5909 MonoDebugMethodInfo *minfo = ctx->minfo;
5910 char *source_file, *dir, *filename;
5911 LLVMValueRef md, args [16], ctx_args [16], md_args [64], type_args [16], ctx_md, type_md;
5919 mono_debug_symfile_get_line_numbers_full (minfo, &source_file, NULL, &n_il_offsets, &il_offsets, &line_numbers, NULL, NULL, NULL, NULL);
5921 source_file = g_strdup ("<unknown>");
5922 dir = g_path_get_dirname (source_file);
5923 filename = g_path_get_basename (source_file);
5925 ctx_args [0] = LLVMConstInt (LLVMInt32Type (), 0x29, FALSE);
5926 args [0] = md_string (filename);
5927 args [1] = md_string (dir);
5928 ctx_args [1] = LLVMMDNode (args, 2);
5929 ctx_md = LLVMMDNode (ctx_args, 2);
5931 type_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subroutine_type, FALSE);
5932 type_args [1] = NULL;
5933 type_args [2] = NULL;
5934 type_args [3] = LLVMMDString ("", 0);
5935 type_args [4] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5936 type_args [5] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
5937 type_args [6] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
5938 type_args [7] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
5939 type_args [8] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5940 type_args [9] = NULL;
5941 type_args [10] = NULL;
5942 type_args [11] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5943 type_args [12] = NULL;
5944 type_args [13] = NULL;
5945 type_args [14] = NULL;
5946 type_md = LLVMMDNode (type_args, 14);
5948 /* http://llvm.org/docs/SourceLevelDebugging.html#subprogram-descriptors */
5949 md_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subprogram, FALSE);
5950 /* Source directory + file pair */
5951 args [0] = md_string (filename);
5952 args [1] = md_string (dir);
5953 md_args [1] = LLVMMDNode (args ,2);
5954 md_args [2] = ctx_md;
5955 md_args [3] = md_string (cfg->method->name);
5956 md_args [4] = md_string (name);
5957 md_args [5] = md_string (name);
5960 md_args [6] = LLVMConstInt (LLVMInt32Type (), line_numbers [0], FALSE);
5962 md_args [6] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5964 md_args [7] = type_md;
5966 md_args [8] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5968 md_args [9] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
5970 md_args [10] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5971 /* Index into a virtual function */
5972 md_args [11] = NULL;
5973 md_args [12] = NULL;
5975 md_args [13] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5977 md_args [14] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
5978 /* Pointer to LLVM function */
5979 md_args [15] = method;
5980 /* Function template parameter */
5981 md_args [16] = NULL;
5982 /* Function declaration descriptor */
5983 md_args [17] = NULL;
5984 /* List of function variables */
5985 md_args [18] = LLVMMDNode (args, 0);
5987 md_args [19] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5988 md = LLVMMDNode (md_args, 20);
5990 if (!module->subprogram_mds)
5991 module->subprogram_mds = g_ptr_array_new ();
5992 g_ptr_array_add (module->subprogram_mds, md);
5996 g_free (source_file);
5997 g_free (il_offsets);
5998 g_free (line_numbers);
6004 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
6006 MonoCompile *cfg = ctx->cfg;
6008 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
6009 MonoDebugSourceLocation *loc;
6010 LLVMValueRef loc_md, md_args [16];
6013 loc = mono_debug_symfile_lookup_location (ctx->minfo, cil_code - cfg->header->code);
6017 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->row, FALSE);
6018 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->column, FALSE);
6019 md_args [nmd_args ++] = ctx->dbg_md;
6020 md_args [nmd_args ++] = NULL;
6021 loc_md = LLVMMDNode (md_args, nmd_args);
6022 LLVMSetCurrentDebugLocation (builder, loc_md);
6023 mono_debug_symfile_free_location (loc);
6030 - Emit LLVM IR from the mono IR using the LLVM C API.
6031 - The original arch specific code remains, so we can fall back to it if we run
6032 into something we can't handle.
6036 A partial list of issues:
6037 - Handling of opcodes which can throw exceptions.
6039 In the mono JIT, these are implemented using code like this:
6046 push throw_pos - method
6047 call <exception trampoline>
6049 The problematic part is push throw_pos - method, which cannot be represented
6050 in the LLVM IR, since it does not support label values.
6051 -> this can be implemented in AOT mode using inline asm + labels, but cannot
6052 be implemented in JIT mode ?
6053 -> a possible but slower implementation would use the normal exception
6054 throwing code but it would need to control the placement of the throw code
6055 (it needs to be exactly after the compare+branch).
6056 -> perhaps add a PC offset intrinsics ?
6058 - efficient implementation of .ovf opcodes.
6060 These are currently implemented as:
6061 <ins which sets the condition codes>
6064 Some overflow opcodes are now supported by LLVM SVN.
6066 - exception handling, unwinding.
6067 - SSA is disabled for methods with exception handlers
6068 - How to obtain unwind info for LLVM compiled methods ?
6069 -> this is now solved by converting the unwind info generated by LLVM
6071 - LLVM uses the c++ exception handling framework, while we use our home grown
6072 code, and couldn't use the c++ one:
6073 - its not supported under VC++, other exotic platforms.
6074 - it might be impossible to support filter clauses with it.
6078 The trampolines need a predictable call sequence, since they need to disasm
6079 the calling code to obtain register numbers / offsets.
6081 LLVM currently generates this code in non-JIT mode:
6082 mov -0x98(%rax),%eax
6084 Here, the vtable pointer is lost.
6085 -> solution: use one vtable trampoline per class.
6087 - passing/receiving the IMT pointer/RGCTX.
6088 -> solution: pass them as normal arguments ?
6092 LLVM does not allow the specification of argument registers etc. This means
6093 that all calls are made according to the platform ABI.
6095 - passing/receiving vtypes.
6097 Vtypes passed/received in registers are handled by the front end by using
6098 a signature with scalar arguments, and loading the parts of the vtype into those
6101 Vtypes passed on the stack are handled using the 'byval' attribute.
6105 Supported though alloca, we need to emit the load/store code.
6109 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
6110 typed registers, so we have to keep track of the precise LLVM type of each vreg.
6111 This is made easier because the IR is already in SSA form.
6112 An additional problem is that our IR is not consistent with types, i.e. i32/ia64
6113 types are frequently used incorrectly.
6118 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then
6119 append the AOT data structures to that file. For methods which cannot be
6120 handled by LLVM, the normal JIT compiled versions are used.
6123 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
6124 * - each bblock should end with a branch
6125 * - setting the return value, making cfg->ret non-volatile
6126 * - avoid some transformations in the JIT which make it harder for us to generate
6128 * - use pointer types to help optimizations.