2 * mini-llvm.c: llvm "Backend" for the mono JIT
4 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
5 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
9 #include <mono/metadata/debug-helpers.h>
10 #include <mono/metadata/debug-mono-symfile.h>
11 #include <mono/metadata/mempool-internals.h>
12 #include <mono/metadata/environment.h>
13 #include <mono/metadata/object-internals.h>
14 #include <mono/utils/mono-tls.h>
15 #include <mono/utils/mono-dl.h>
16 #include <mono/utils/mono-time.h>
17 #include <mono/utils/freebsd-dwarf.h>
19 #ifndef __STDC_LIMIT_MACROS
20 #define __STDC_LIMIT_MACROS
22 #ifndef __STDC_CONSTANT_MACROS
23 #define __STDC_CONSTANT_MACROS
26 #include "llvm-c/Core.h"
27 #include "llvm-c/ExecutionEngine.h"
28 #include "llvm-c/BitWriter.h"
29 #include "llvm-c/Analysis.h"
31 #include "mini-llvm-cpp.h"
36 extern void *memset(void *, int, size_t);
37 void bzero (void *to, size_t count) { memset (to, 0, count); }
41 #if LLVM_API_VERSION < 4
42 #error "The version of the mono llvm repository is too old."
46 * Information associated by mono with LLVM modules.
49 LLVMModuleRef lmodule;
50 LLVMValueRef throw_icall, rethrow, match_exc, throw_corlib_exception, resume_eh;
51 GHashTable *llvm_types;
53 const char *got_symbol;
54 const char *get_method_symbol;
55 const char *get_unbox_tramp_symbol;
56 GHashTable *plt_entries;
57 GHashTable *plt_entries_ji;
58 GHashTable *method_to_lmethod;
59 GHashTable *direct_callables;
64 GPtrArray *subprogram_mds;
66 LLVMExecutionEngineRef ee;
67 gboolean external_symbols;
72 MonoAssembly *assembly;
74 MonoAotFileInfo aot_info;
75 const char *jit_got_symbol;
76 const char *eh_frame_symbol;
77 LLVMValueRef get_method, get_unbox_tramp;
78 LLVMValueRef init_method, init_method_gshared_rgctx, init_method_gshared_this;
79 LLVMValueRef code_start, code_end;
80 LLVMValueRef inited_var;
81 int max_inited_idx, max_method_idx;
82 gboolean has_jitted_code;
85 GHashTable *idx_to_lmethod;
86 GHashTable *idx_to_unbox_tramp;
87 /* Maps a MonoMethod to LLVM instructions representing it */
88 GHashTable *method_to_callers;
89 LLVMContextRef context;
90 LLVMValueRef sentinel_exception;
94 * Information associated by the backend with mono basic blocks.
97 LLVMBasicBlockRef bblock, end_bblock;
98 LLVMValueRef finally_ind;
99 gboolean added, invoke_target;
101 * If this bblock is the start of a finally clause, this is a list of bblocks it
102 * needs to branch to in ENDFINALLY.
104 GSList *call_handler_return_bbs;
106 * If this bblock is the start of a finally clause, this is the bblock that
107 * CALL_HANDLER needs to branch to.
109 LLVMBasicBlockRef call_handler_target_bb;
110 /* The list of switch statements generated by ENDFINALLY instructions */
111 GSList *endfinally_switch_ins_list;
116 * Structure containing emit state
119 MonoMemPool *mempool;
121 /* Maps method names to the corresponding LLVMValueRef */
122 GHashTable *emitted_method_decls;
125 LLVMValueRef lmethod;
126 MonoLLVMModule *module;
127 LLVMModuleRef lmodule;
129 int sindex, default_index, ex_index;
130 LLVMBuilderRef builder;
131 LLVMValueRef *values, *addresses;
132 MonoType **vreg_cli_types;
134 MonoMethodSignature *sig;
136 GHashTable *region_to_handler;
137 GHashTable *clause_to_handler;
138 LLVMBuilderRef alloca_builder;
139 LLVMValueRef last_alloca;
140 LLVMValueRef rgctx_arg;
141 LLVMValueRef this_arg;
142 LLVMTypeRef *vreg_types;
143 LLVMBasicBlockRef init_bb, inited_bb;
145 gboolean *unreachable;
147 gboolean has_got_access;
148 int this_arg_pindex, rgctx_arg_pindex;
149 LLVMValueRef imt_rgctx_loc;
150 GHashTable *llvm_types;
152 MonoDebugMethodInfo *minfo;
154 /* For every clause, the clauses it is nested in */
157 GHashTable *exc_meta;
158 GHashTable *method_to_callers;
164 MonoBasicBlock *in_bb;
169 * Instruction metadata
170 * This is the same as ins_info, but LREG != IREG.
178 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
179 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
186 /* keep in sync with the enum in mini.h */
189 #include "mini-ops.h"
194 #if SIZEOF_VOID_P == 4
195 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
197 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
200 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
203 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
205 #define TRACE_FAILURE(msg)
209 #define IS_TARGET_X86 1
211 #define IS_TARGET_X86 0
215 #define IS_TARGET_AMD64 1
217 #define IS_TARGET_AMD64 0
220 #define LLVM_FAILURE(ctx, reason) do { \
221 TRACE_FAILURE (reason); \
222 (ctx)->cfg->exception_message = g_strdup (reason); \
223 (ctx)->cfg->disable_llvm = TRUE; \
227 #define CHECK_FAILURE(ctx) do { \
228 if ((ctx)->cfg->disable_llvm) \
232 static LLVMIntPredicate cond_to_llvm_cond [] = {
245 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
258 static MonoNativeTlsKey current_cfg_tls_id;
260 static MonoLLVMModule aot_module;
261 static int memset_param_count, memcpy_param_count;
262 static const char *memset_func_name;
263 static const char *memcpy_func_name;
265 static void init_jit_module (MonoDomain *domain);
267 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
268 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
269 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
274 * The LLVM type with width == sizeof (gpointer)
279 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
285 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
291 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
297 * Return the size of the LLVM representation of the vtype T.
300 get_vtype_size (MonoType *t)
304 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
306 /* LLVMArgAsIArgs depends on this since it stores whole words */
307 while (size < 2 * sizeof (gpointer) && mono_is_power_of_two (size) == -1)
314 * simd_class_to_llvm_type:
316 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
319 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
321 if (!strcmp (klass->name, "Vector2d")) {
322 return LLVMVectorType (LLVMDoubleType (), 2);
323 } else if (!strcmp (klass->name, "Vector2l")) {
324 return LLVMVectorType (LLVMInt64Type (), 2);
325 } else if (!strcmp (klass->name, "Vector2ul")) {
326 return LLVMVectorType (LLVMInt64Type (), 2);
327 } else if (!strcmp (klass->name, "Vector4i")) {
328 return LLVMVectorType (LLVMInt32Type (), 4);
329 } else if (!strcmp (klass->name, "Vector4ui")) {
330 return LLVMVectorType (LLVMInt32Type (), 4);
331 } else if (!strcmp (klass->name, "Vector4f")) {
332 return LLVMVectorType (LLVMFloatType (), 4);
333 } else if (!strcmp (klass->name, "Vector8s")) {
334 return LLVMVectorType (LLVMInt16Type (), 8);
335 } else if (!strcmp (klass->name, "Vector8us")) {
336 return LLVMVectorType (LLVMInt16Type (), 8);
337 } else if (!strcmp (klass->name, "Vector16sb")) {
338 return LLVMVectorType (LLVMInt8Type (), 16);
339 } else if (!strcmp (klass->name, "Vector16b")) {
340 return LLVMVectorType (LLVMInt8Type (), 16);
342 printf ("%s\n", klass->name);
348 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
349 static inline G_GNUC_UNUSED LLVMTypeRef
350 type_to_simd_type (int type)
354 return LLVMVectorType (LLVMInt8Type (), 16);
356 return LLVMVectorType (LLVMInt16Type (), 8);
358 return LLVMVectorType (LLVMInt32Type (), 4);
360 return LLVMVectorType (LLVMInt64Type (), 2);
362 return LLVMVectorType (LLVMDoubleType (), 2);
364 return LLVMVectorType (LLVMFloatType (), 4);
366 g_assert_not_reached ();
372 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
374 int i, size, nfields, esize;
375 LLVMTypeRef *eltypes;
380 t = &klass->byval_arg;
382 if (mini_type_is_hfa (t, &nfields, &esize)) {
384 * This is needed on arm64 where HFAs are returned in
388 eltypes = g_new (LLVMTypeRef, size);
389 for (i = 0; i < size; ++i)
390 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
392 size = get_vtype_size (t);
394 eltypes = g_new (LLVMTypeRef, size);
395 for (i = 0; i < size; ++i)
396 eltypes [i] = LLVMInt8Type ();
399 name = mono_type_full_name (&klass->byval_arg);
400 ltype = LLVMStructCreateNamed (module->context, name);
401 LLVMStructSetBody (ltype, eltypes, size, FALSE);
411 * Return the LLVM type corresponding to T.
414 type_to_llvm_type (EmitContext *ctx, MonoType *t)
417 return LLVMPointerType (LLVMInt8Type (), 0);
419 t = mini_get_underlying_type (t);
422 return LLVMVoidType ();
424 return LLVMInt8Type ();
426 return LLVMInt16Type ();
428 return LLVMInt32Type ();
430 return LLVMInt8Type ();
432 return LLVMInt16Type ();
434 return LLVMInt32Type ();
435 case MONO_TYPE_BOOLEAN:
436 return LLVMInt8Type ();
439 return LLVMInt64Type ();
441 return LLVMInt16Type ();
443 return LLVMFloatType ();
445 return LLVMDoubleType ();
448 return IntPtrType ();
449 case MONO_TYPE_OBJECT:
450 case MONO_TYPE_CLASS:
451 case MONO_TYPE_ARRAY:
452 case MONO_TYPE_SZARRAY:
453 case MONO_TYPE_STRING:
455 return ObjRefType ();
458 /* Because of generic sharing */
459 return ObjRefType ();
460 case MONO_TYPE_GENERICINST:
461 if (!mono_type_generic_inst_is_valuetype (t))
462 return ObjRefType ();
464 case MONO_TYPE_VALUETYPE:
465 case MONO_TYPE_TYPEDBYREF: {
469 klass = mono_class_from_mono_type (t);
471 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
472 return simd_class_to_llvm_type (ctx, klass);
475 return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
477 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
479 ltype = create_llvm_type_for_type (ctx->module, klass);
480 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
486 printf ("X: %d\n", t->type);
487 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
488 ctx->cfg->disable_llvm = TRUE;
496 * Return whenever T is an unsigned int type.
499 type_is_unsigned (EmitContext *ctx, MonoType *t)
516 * type_to_llvm_arg_type:
518 * Same as type_to_llvm_type, but treat i8/i16 as i32.
521 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
523 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
526 * This works on all abis except arm64/ios which passes multiple
527 * arguments in one stack slot.
530 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
532 * LLVM generates code which only sets the lower bits, while JITted
533 * code expects all the bits to be set.
535 ptype = LLVMInt32Type ();
543 * llvm_type_to_stack_type:
545 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
548 static G_GNUC_UNUSED LLVMTypeRef
549 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
553 if (type == LLVMInt8Type ())
554 return LLVMInt32Type ();
555 else if (type == LLVMInt16Type ())
556 return LLVMInt32Type ();
557 else if (!cfg->r4fp && type == LLVMFloatType ())
558 return LLVMDoubleType ();
564 * regtype_to_llvm_type:
566 * Return the LLVM type corresponding to the regtype C used in instruction
570 regtype_to_llvm_type (char c)
574 return LLVMInt32Type ();
576 return LLVMInt64Type ();
578 return LLVMDoubleType ();
587 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
590 op_to_llvm_type (int opcode)
595 return LLVMInt8Type ();
598 return LLVMInt8Type ();
601 return LLVMInt16Type ();
604 return LLVMInt16Type ();
607 return LLVMInt32Type ();
610 return LLVMInt32Type ();
612 return LLVMInt64Type ();
614 return LLVMFloatType ();
616 return LLVMDoubleType ();
618 return LLVMInt64Type ();
620 return LLVMInt32Type ();
622 return LLVMInt64Type ();
627 return LLVMInt8Type ();
632 return LLVMInt16Type ();
635 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
642 return LLVMInt32Type ();
649 return LLVMInt64Type ();
651 printf ("%s\n", mono_inst_name (opcode));
652 g_assert_not_reached ();
657 #define CLAUSE_START(clause) ((clause)->try_offset)
658 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
661 * load_store_to_llvm_type:
663 * Return the size/sign/zero extension corresponding to the load/store opcode
667 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
673 case OP_LOADI1_MEMBASE:
674 case OP_STOREI1_MEMBASE_REG:
675 case OP_STOREI1_MEMBASE_IMM:
676 case OP_ATOMIC_LOAD_I1:
677 case OP_ATOMIC_STORE_I1:
680 return LLVMInt8Type ();
681 case OP_LOADU1_MEMBASE:
683 case OP_ATOMIC_LOAD_U1:
684 case OP_ATOMIC_STORE_U1:
687 return LLVMInt8Type ();
688 case OP_LOADI2_MEMBASE:
689 case OP_STOREI2_MEMBASE_REG:
690 case OP_STOREI2_MEMBASE_IMM:
691 case OP_ATOMIC_LOAD_I2:
692 case OP_ATOMIC_STORE_I2:
695 return LLVMInt16Type ();
696 case OP_LOADU2_MEMBASE:
698 case OP_ATOMIC_LOAD_U2:
699 case OP_ATOMIC_STORE_U2:
702 return LLVMInt16Type ();
703 case OP_LOADI4_MEMBASE:
704 case OP_LOADU4_MEMBASE:
707 case OP_STOREI4_MEMBASE_REG:
708 case OP_STOREI4_MEMBASE_IMM:
709 case OP_ATOMIC_LOAD_I4:
710 case OP_ATOMIC_STORE_I4:
711 case OP_ATOMIC_LOAD_U4:
712 case OP_ATOMIC_STORE_U4:
714 return LLVMInt32Type ();
715 case OP_LOADI8_MEMBASE:
717 case OP_STOREI8_MEMBASE_REG:
718 case OP_STOREI8_MEMBASE_IMM:
719 case OP_ATOMIC_LOAD_I8:
720 case OP_ATOMIC_STORE_I8:
721 case OP_ATOMIC_LOAD_U8:
722 case OP_ATOMIC_STORE_U8:
724 return LLVMInt64Type ();
725 case OP_LOADR4_MEMBASE:
726 case OP_STORER4_MEMBASE_REG:
727 case OP_ATOMIC_LOAD_R4:
728 case OP_ATOMIC_STORE_R4:
730 return LLVMFloatType ();
731 case OP_LOADR8_MEMBASE:
732 case OP_STORER8_MEMBASE_REG:
733 case OP_ATOMIC_LOAD_R8:
734 case OP_ATOMIC_STORE_R8:
736 return LLVMDoubleType ();
737 case OP_LOAD_MEMBASE:
739 case OP_STORE_MEMBASE_REG:
740 case OP_STORE_MEMBASE_IMM:
741 *size = sizeof (gpointer);
742 return IntPtrType ();
744 g_assert_not_reached ();
752 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
755 ovf_op_to_intrins (int opcode)
759 return "llvm.sadd.with.overflow.i32";
761 return "llvm.uadd.with.overflow.i32";
763 return "llvm.ssub.with.overflow.i32";
765 return "llvm.usub.with.overflow.i32";
767 return "llvm.smul.with.overflow.i32";
769 return "llvm.umul.with.overflow.i32";
771 return "llvm.sadd.with.overflow.i64";
773 return "llvm.uadd.with.overflow.i64";
775 return "llvm.ssub.with.overflow.i64";
777 return "llvm.usub.with.overflow.i64";
779 return "llvm.smul.with.overflow.i64";
781 return "llvm.umul.with.overflow.i64";
783 g_assert_not_reached ();
789 simd_op_to_intrins (int opcode)
792 #if defined(TARGET_X86) || defined(TARGET_AMD64)
794 return "llvm.x86.sse2.min.pd";
796 return "llvm.x86.sse.min.ps";
798 return "llvm.x86.sse41.pminud";
800 return "llvm.x86.sse41.pminuw";
802 return "llvm.x86.sse2.pminu.b";
804 return "llvm.x86.sse2.pmins.w";
806 return "llvm.x86.sse2.max.pd";
808 return "llvm.x86.sse.max.ps";
810 return "llvm.x86.sse3.hadd.pd";
812 return "llvm.x86.sse3.hadd.ps";
814 return "llvm.x86.sse3.hsub.pd";
816 return "llvm.x86.sse3.hsub.ps";
818 return "llvm.x86.sse41.pmaxud";
820 return "llvm.x86.sse41.pmaxuw";
822 return "llvm.x86.sse2.pmaxu.b";
824 return "llvm.x86.sse3.addsub.ps";
826 return "llvm.x86.sse3.addsub.pd";
827 case OP_EXTRACT_MASK:
828 return "llvm.x86.sse2.pmovmskb.128";
831 return "llvm.x86.sse2.psrli.w";
834 return "llvm.x86.sse2.psrli.d";
837 return "llvm.x86.sse2.psrli.q";
840 return "llvm.x86.sse2.pslli.w";
843 return "llvm.x86.sse2.pslli.d";
846 return "llvm.x86.sse2.pslli.q";
849 return "llvm.x86.sse2.psrai.w";
852 return "llvm.x86.sse2.psrai.d";
854 return "llvm.x86.sse2.padds.b";
856 return "llvm.x86.sse2.padds.w";
858 return "llvm.x86.sse2.psubs.b";
860 return "llvm.x86.sse2.psubs.w";
861 case OP_PADDB_SAT_UN:
862 return "llvm.x86.sse2.paddus.b";
863 case OP_PADDW_SAT_UN:
864 return "llvm.x86.sse2.paddus.w";
865 case OP_PSUBB_SAT_UN:
866 return "llvm.x86.sse2.psubus.b";
867 case OP_PSUBW_SAT_UN:
868 return "llvm.x86.sse2.psubus.w";
870 return "llvm.x86.sse2.pavg.b";
872 return "llvm.x86.sse2.pavg.w";
874 return "llvm.x86.sse.sqrt.ps";
876 return "llvm.x86.sse2.sqrt.pd";
878 return "llvm.x86.sse.rsqrt.ps";
880 return "llvm.x86.sse.rcp.ps";
882 return "llvm.x86.sse2.cvtdq2pd";
884 return "llvm.x86.sse2.cvtdq2ps";
886 return "llvm.x86.sse2.cvtpd2dq";
888 return "llvm.x86.sse2.cvtps2dq";
890 return "llvm.x86.sse2.cvtpd2ps";
892 return "llvm.x86.sse2.cvtps2pd";
894 return "llvm.x86.sse2.cvttpd2dq";
896 return "llvm.x86.sse2.cvttps2dq";
898 return "llvm.x86.sse.cmp.ps";
900 return "llvm.x86.sse2.cmp.pd";
902 return "llvm.x86.sse2.packsswb.128";
904 return "llvm.x86.sse2.packssdw.128";
906 return "llvm.x86.sse2.packuswb.128";
908 return "llvm.x86.sse41.packusdw";
910 return "llvm.x86.sse2.pmulh.w";
911 case OP_PMULW_HIGH_UN:
912 return "llvm.x86.sse2.pmulhu.w";
915 g_assert_not_reached ();
921 simd_op_to_llvm_type (int opcode)
923 #if defined(TARGET_X86) || defined(TARGET_AMD64)
927 return type_to_simd_type (MONO_TYPE_R8);
930 return type_to_simd_type (MONO_TYPE_I8);
933 return type_to_simd_type (MONO_TYPE_I4);
938 return type_to_simd_type (MONO_TYPE_I2);
942 return type_to_simd_type (MONO_TYPE_I1);
944 return type_to_simd_type (MONO_TYPE_R4);
947 return type_to_simd_type (MONO_TYPE_I4);
951 return type_to_simd_type (MONO_TYPE_R8);
955 return type_to_simd_type (MONO_TYPE_R4);
956 case OP_EXTRACT_MASK:
957 return type_to_simd_type (MONO_TYPE_I1);
963 return type_to_simd_type (MONO_TYPE_R4);
966 return type_to_simd_type (MONO_TYPE_R8);
968 g_assert_not_reached ();
979 * Return the LLVM basic block corresponding to BB.
981 static LLVMBasicBlockRef
982 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
984 char bb_name_buf [128];
987 if (ctx->bblocks [bb->block_num].bblock == NULL) {
988 if (bb->flags & BB_EXCEPTION_HANDLER) {
989 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
990 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
991 bb_name = bb_name_buf;
992 } else if (bb->block_num < 256) {
993 if (!ctx->module->bb_names) {
994 ctx->module->bb_names_len = 256;
995 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
997 if (!ctx->module->bb_names [bb->block_num]) {
1000 n = g_strdup_printf ("BB%d", bb->block_num);
1001 mono_memory_barrier ();
1002 ctx->module->bb_names [bb->block_num] = n;
1004 bb_name = ctx->module->bb_names [bb->block_num];
1006 sprintf (bb_name_buf, "BB%d", bb->block_num);
1007 bb_name = bb_name_buf;
1010 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1011 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1014 return ctx->bblocks [bb->block_num].bblock;
1020 * Return the last LLVM bblock corresponding to BB.
1021 * This might not be equal to the bb returned by get_bb () since we need to generate
1022 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1024 static LLVMBasicBlockRef
1025 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1028 return ctx->bblocks [bb->block_num].end_bblock;
1031 static LLVMBasicBlockRef
1032 gen_bb (EmitContext *ctx, const char *prefix)
1036 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1037 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1043 * Return the target of the patch identified by TYPE and TARGET.
1046 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1050 memset (&ji, 0, sizeof (ji));
1052 ji.data.target = target;
1054 return mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
1060 * Emit code to convert the LLVM value V to DTYPE.
1063 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1065 LLVMTypeRef stype = LLVMTypeOf (v);
1067 if (stype != dtype) {
1068 gboolean ext = FALSE;
1071 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1073 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1075 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1079 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1081 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1082 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1085 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1086 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1087 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1088 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1089 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1090 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1091 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1092 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1094 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1095 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1096 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1097 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1098 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1099 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1101 if (mono_arch_is_soft_float ()) {
1102 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1103 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1104 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1105 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1108 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1109 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1112 LLVMDumpValue (LLVMConstNull (dtype));
1113 g_assert_not_reached ();
1121 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1123 return convert_full (ctx, v, dtype, FALSE);
1127 * emit_volatile_load:
1129 * If vreg is volatile, emit a load from its address.
1132 emit_volatile_load (EmitContext *ctx, int vreg)
1136 LLVMValueRef v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1137 t = ctx->vreg_cli_types [vreg];
1138 if (t && !t->byref) {
1140 * Might have to zero extend since llvm doesn't have
1143 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1144 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1145 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1146 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1147 else if (t->type == MONO_TYPE_U8)
1148 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1155 * emit_volatile_store:
1157 * If VREG is volatile, emit a store from its value to its address.
1160 emit_volatile_store (EmitContext *ctx, int vreg)
1162 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1164 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1165 g_assert (ctx->addresses [vreg]);
1166 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1171 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1173 LLVMTypeRef ret_type;
1174 LLVMTypeRef *param_types = NULL;
1179 rtype = mini_get_underlying_type (sig->ret);
1180 ret_type = type_to_llvm_type (ctx, rtype);
1181 CHECK_FAILURE (ctx);
1183 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1187 param_types [pindex ++] = ThisType ();
1188 for (i = 0; i < sig->param_count; ++i)
1189 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1191 CHECK_FAILURE (ctx);
1193 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1194 g_free (param_types);
1199 g_free (param_types);
1205 * sig_to_llvm_sig_full:
1207 * Return the LLVM signature corresponding to the mono signature SIG using the
1208 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1211 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1213 LLVMTypeRef ret_type;
1214 LLVMTypeRef *param_types = NULL;
1216 int i, j, pindex, vret_arg_pindex = 0;
1217 gboolean vretaddr = FALSE;
1221 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1223 rtype = mini_get_underlying_type (sig->ret);
1224 ret_type = type_to_llvm_type (ctx, rtype);
1225 CHECK_FAILURE (ctx);
1227 switch (cinfo->ret.storage) {
1228 case LLVMArgVtypeInReg:
1229 /* LLVM models this by returning an aggregate value */
1230 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1231 LLVMTypeRef members [2];
1233 members [0] = IntPtrType ();
1234 ret_type = LLVMStructType (members, 1, FALSE);
1235 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1237 ret_type = LLVMVoidType ();
1238 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1239 LLVMTypeRef members [2];
1241 members [0] = IntPtrType ();
1242 members [1] = IntPtrType ();
1243 ret_type = LLVMStructType (members, 2, FALSE);
1245 g_assert_not_reached ();
1248 case LLVMArgVtypeByVal:
1249 /* Vtype returned normally by val */
1251 case LLVMArgVtypeAsScalar:
1252 /* LLVM models this by returning an int */
1253 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1254 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (mgreg_t) * 8);
1256 case LLVMArgFpStruct: {
1257 /* Vtype returned as a fp struct */
1258 LLVMTypeRef members [16];
1260 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1261 for (i = 0; i < cinfo->ret.nslots; ++i)
1262 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1263 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1266 case LLVMArgVtypeByRef:
1267 /* Vtype returned using a hidden argument */
1268 ret_type = LLVMVoidType ();
1270 case LLVMArgVtypeRetAddr:
1271 case LLVMArgScalarRetAddr:
1273 ret_type = LLVMVoidType ();
1279 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1281 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1283 * Has to be the first argument because of the sret argument attribute
1284 * FIXME: This might conflict with passing 'this' as the first argument, but
1285 * this is only used on arm64 which has a dedicated struct return register.
1287 cinfo->vret_arg_pindex = pindex;
1288 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1289 CHECK_FAILURE (ctx);
1290 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1293 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1294 cinfo->rgctx_arg_pindex = pindex;
1295 param_types [pindex] = ctx->module->ptr_type;
1298 if (cinfo->imt_arg) {
1299 cinfo->imt_arg_pindex = pindex;
1300 param_types [pindex] = ctx->module->ptr_type;
1304 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1305 vret_arg_pindex = pindex;
1306 if (cinfo->vret_arg_index == 1) {
1307 /* Add the slots consumed by the first argument */
1308 LLVMArgInfo *ainfo = &cinfo->args [0];
1309 switch (ainfo->storage) {
1310 case LLVMArgVtypeInReg:
1311 for (j = 0; j < 2; ++j) {
1312 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1321 cinfo->vret_arg_pindex = vret_arg_pindex;
1324 if (vretaddr && vret_arg_pindex == pindex)
1325 param_types [pindex ++] = IntPtrType ();
1327 cinfo->this_arg_pindex = pindex;
1328 param_types [pindex ++] = ThisType ();
1330 if (vretaddr && vret_arg_pindex == pindex)
1331 param_types [pindex ++] = IntPtrType ();
1332 for (i = 0; i < sig->param_count; ++i) {
1333 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1335 if (vretaddr && vret_arg_pindex == pindex)
1336 param_types [pindex ++] = IntPtrType ();
1337 ainfo->pindex = pindex;
1339 switch (ainfo->storage) {
1340 case LLVMArgVtypeInReg:
1341 for (j = 0; j < 2; ++j) {
1342 switch (ainfo->pair_storage [j]) {
1344 param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
1349 g_assert_not_reached ();
1353 case LLVMArgVtypeByVal:
1354 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1355 CHECK_FAILURE (ctx);
1356 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1359 case LLVMArgAsIArgs:
1360 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1363 case LLVMArgVtypeByRef:
1364 case LLVMArgScalarByRef:
1365 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1366 CHECK_FAILURE (ctx);
1367 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1370 case LLVMArgAsFpArgs: {
1373 for (j = 0; j < ainfo->nslots; ++j)
1374 param_types [pindex + j] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1375 pindex += ainfo->nslots;
1378 case LLVMArgVtypeAsScalar:
1379 g_assert_not_reached ();
1382 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1386 if (vretaddr && vret_arg_pindex == pindex)
1387 param_types [pindex ++] = IntPtrType ();
1388 if (ctx->llvm_only && cinfo->rgctx_arg) {
1389 /* Pass the rgctx as the last argument */
1390 cinfo->rgctx_arg_pindex = pindex;
1391 param_types [pindex] = ctx->module->ptr_type;
1395 CHECK_FAILURE (ctx);
1397 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1398 g_free (param_types);
1403 g_free (param_types);
1409 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1411 return sig_to_llvm_sig_full (ctx, sig, NULL);
1415 * LLVMFunctionType1:
1417 * Create an LLVM function type from the arguments.
1419 static G_GNUC_UNUSED LLVMTypeRef
1420 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1423 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1427 * LLVMFunctionType1:
1429 * Create an LLVM function type from the arguments.
1431 static G_GNUC_UNUSED LLVMTypeRef
1432 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1433 LLVMTypeRef ParamType1,
1436 LLVMTypeRef param_types [1];
1438 param_types [0] = ParamType1;
1440 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1444 * LLVMFunctionType2:
1446 * Create an LLVM function type from the arguments.
1448 static G_GNUC_UNUSED LLVMTypeRef
1449 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1450 LLVMTypeRef ParamType1,
1451 LLVMTypeRef ParamType2,
1454 LLVMTypeRef param_types [2];
1456 param_types [0] = ParamType1;
1457 param_types [1] = ParamType2;
1459 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1463 * LLVMFunctionType3:
1465 * Create an LLVM function type from the arguments.
1467 static G_GNUC_UNUSED LLVMTypeRef
1468 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1469 LLVMTypeRef ParamType1,
1470 LLVMTypeRef ParamType2,
1471 LLVMTypeRef ParamType3,
1474 LLVMTypeRef param_types [3];
1476 param_types [0] = ParamType1;
1477 param_types [1] = ParamType2;
1478 param_types [2] = ParamType3;
1480 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1486 * Create an LLVM builder and remember it so it can be freed later.
1488 static LLVMBuilderRef
1489 create_builder (EmitContext *ctx)
1491 LLVMBuilderRef builder = LLVMCreateBuilder ();
1493 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1499 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1504 case MONO_PATCH_INFO_INTERNAL_METHOD:
1505 name = g_strdup_printf ("jit_icall_%s", data);
1508 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1516 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1520 LLVMValueRef indexes [2];
1522 LLVMValueRef got_entry_addr, load;
1523 LLVMBuilderRef builder = ctx->builder;
1528 ji = g_new0 (MonoJumpInfo, 1);
1530 ji->data.target = data;
1532 ji = mono_aot_patch_info_dup (ji);
1534 ji->next = cfg->patch_info;
1535 cfg->patch_info = ji;
1537 got_offset = mono_aot_get_got_offset (cfg->patch_info);
1538 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1540 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1541 * explicitly initialize it.
1543 if (!mono_aot_is_shared_got_offset (got_offset)) {
1544 //mono_print_ji (ji);
1546 ctx->has_got_access = TRUE;
1549 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1550 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1551 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1553 name = get_aotconst_name (type, data, got_offset);
1554 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1556 //set_invariant_load_flag (load);
1562 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1564 LLVMValueRef callee;
1566 if (ctx->llvm_only) {
1569 callee_name = mono_aot_get_direct_call_symbol (type, data);
1571 /* Directly callable */
1573 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1575 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1577 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1579 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1581 g_free (callee_name);
1587 * Calls are made through the GOT.
1589 load = get_aotconst (ctx, type, data);
1591 return convert (ctx, load, LLVMPointerType (llvm_sig, 0));
1593 MonoJumpInfo *ji = NULL;
1595 callee_name = mono_aot_get_plt_symbol (type, data);
1599 if (ctx->cfg->compile_aot)
1600 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1601 mono_add_patch_info (ctx->cfg, 0, type, data);
1604 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
1606 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1608 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1610 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
1613 if (ctx->cfg->compile_aot) {
1614 ji = g_new0 (MonoJumpInfo, 1);
1616 ji->data.target = data;
1618 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
1626 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
1628 MonoMethodHeader *header = cfg->header;
1629 MonoExceptionClause *clause;
1633 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
1634 return (bb->region >> 8) - 1;
1637 for (i = 0; i < header->num_clauses; ++i) {
1638 clause = &header->clauses [i];
1640 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
1647 static MonoExceptionClause *
1648 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
1650 // Since they're sorted by nesting we just need
1651 // the first one that the bb is a member of
1652 MonoExceptionClause *last = NULL;
1654 for (int i = 0; i < cfg->header->num_clauses; i++) {
1655 MonoExceptionClause *curr = &cfg->header->clauses [i];
1657 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
1660 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset)) {
1661 if (last && CLAUSE_END(last) > CLAUSE_END(curr))
1675 set_metadata_flag (LLVMValueRef v, const char *flag_name)
1677 LLVMValueRef md_arg;
1680 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1681 md_arg = LLVMMDString ("mono", 4);
1682 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1686 set_invariant_load_flag (LLVMValueRef v)
1688 LLVMValueRef md_arg;
1690 const char *flag_name;
1692 // FIXME: Cache this
1693 flag_name = "invariant.load";
1694 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1695 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
1696 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1702 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
1706 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
1708 MonoCompile *cfg = ctx->cfg;
1709 LLVMValueRef lcall = NULL;
1710 LLVMBuilderRef builder = *builder_ref;
1711 MonoExceptionClause *clause;
1713 if (ctx->llvm_only) {
1714 clause = get_most_deep_clause (cfg, ctx, bb);
1717 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1720 * Have to use an invoke instead of a call, branching to the
1721 * handler bblock of the clause containing this bblock.
1723 intptr_t key = CLAUSE_END(clause);
1725 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
1727 // FIXME: Find the one that has the lowest end bound for the right start address
1728 // FIXME: Finally + nesting
1731 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
1734 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
1736 builder = ctx->builder = create_builder (ctx);
1737 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1739 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1743 int clause_index = get_handler_clause (cfg, bb);
1745 if (clause_index != -1) {
1746 MonoMethodHeader *header = cfg->header;
1747 MonoExceptionClause *ec = &header->clauses [clause_index];
1748 MonoBasicBlock *tblock;
1749 LLVMBasicBlockRef ex_bb, noex_bb;
1752 * Have to use an invoke instead of a call, branching to the
1753 * handler bblock of the clause containing this bblock.
1756 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1758 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
1761 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
1763 ex_bb = get_bb (ctx, tblock);
1765 noex_bb = gen_bb (ctx, "NOEX_BB");
1768 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1770 builder = ctx->builder = create_builder (ctx);
1771 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1773 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1778 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1779 ctx->builder = builder;
1783 *builder_ref = ctx->builder;
1789 emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting, BarrierKind barrier)
1791 const char *intrins_name;
1792 LLVMValueRef args [16], res;
1793 LLVMTypeRef addr_type;
1795 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1796 LLVMAtomicOrdering ordering;
1799 case LLVM_BARRIER_NONE:
1800 ordering = LLVMAtomicOrderingNotAtomic;
1802 case LLVM_BARRIER_ACQ:
1803 ordering = LLVMAtomicOrderingAcquire;
1805 case LLVM_BARRIER_SEQ:
1806 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1809 g_assert_not_reached ();
1814 * We handle loads which can fault by calling a mono specific intrinsic
1815 * using an invoke, so they are handled properly inside try blocks.
1816 * We can't use this outside clauses, since LLVM optimizes intrinsics which
1817 * are marked with IntrReadArgMem.
1821 intrins_name = "llvm.mono.load.i8.p0i8";
1824 intrins_name = "llvm.mono.load.i16.p0i16";
1827 intrins_name = "llvm.mono.load.i32.p0i32";
1830 intrins_name = "llvm.mono.load.i64.p0i64";
1833 g_assert_not_reached ();
1836 addr_type = LLVMTypeOf (addr);
1837 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
1838 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1841 args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1842 args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1843 args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1844 res = emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->lmodule, intrins_name), args, 4);
1846 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
1847 res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
1848 else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
1849 res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
1856 * We emit volatile loads for loads which can fault, because otherwise
1857 * LLVM will generate invalid code when encountering a load from a
1860 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting, barrier);
1862 /* Mark it with a custom metadata */
1865 set_metadata_flag (res, "mono.faulting.load");
1873 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
1875 return emit_load_general (ctx, bb, builder_ref, size, addr, name, is_faulting, LLVM_BARRIER_NONE);
1879 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting, BarrierKind barrier)
1881 const char *intrins_name;
1882 LLVMValueRef args [16];
1884 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1885 LLVMAtomicOrdering ordering;
1888 case LLVM_BARRIER_NONE:
1889 ordering = LLVMAtomicOrderingNotAtomic;
1891 case LLVM_BARRIER_REL:
1892 ordering = LLVMAtomicOrderingRelease;
1894 case LLVM_BARRIER_SEQ:
1895 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1898 g_assert_not_reached ();
1904 intrins_name = "llvm.mono.store.i8.p0i8";
1907 intrins_name = "llvm.mono.store.i16.p0i16";
1910 intrins_name = "llvm.mono.store.i32.p0i32";
1913 intrins_name = "llvm.mono.store.i64.p0i64";
1916 g_assert_not_reached ();
1919 if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
1920 value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
1921 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1926 args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1927 args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1928 args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1929 emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->lmodule, intrins_name), args, 5);
1931 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
1936 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting)
1938 emit_store_general (ctx, bb, builder_ref, size, value, addr, is_faulting, LLVM_BARRIER_NONE);
1942 * emit_cond_system_exception:
1944 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
1945 * Might set the ctx exception.
1948 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
1950 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
1951 LLVMBuilderRef builder;
1952 MonoClass *exc_class;
1953 LLVMValueRef args [2];
1954 LLVMValueRef callee;
1956 ex_bb = gen_bb (ctx, "EX_BB");
1958 ex2_bb = gen_bb (ctx, "EX2_BB");
1959 noex_bb = gen_bb (ctx, "NOEX_BB");
1961 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
1963 exc_class = mono_class_from_name (mono_get_corlib (), "System", exc_type);
1964 g_assert (exc_class);
1966 /* Emit exception throwing code */
1967 ctx->builder = builder = create_builder (ctx);
1968 LLVMPositionBuilderAtEnd (builder, ex_bb);
1970 if (ctx->cfg->llvm_only) {
1971 static LLVMTypeRef sig;
1974 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
1975 callee = get_callee (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_llvm_throw_corlib_exception");
1977 LLVMBuildBr (builder, ex2_bb);
1979 ctx->builder = builder = create_builder (ctx);
1980 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
1982 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
1983 emit_call (ctx, bb, &builder, callee, args, 1);
1984 LLVMBuildUnreachable (builder);
1986 ctx->builder = builder = create_builder (ctx);
1987 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1989 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1995 callee = ctx->module->throw_corlib_exception;
1998 const char *icall_name;
2000 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2001 icall_name = "llvm_throw_corlib_exception_abs_trampoline";
2003 if (ctx->cfg->compile_aot) {
2004 callee = get_callee (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
2006 callee = LLVMAddFunction (ctx->lmodule, "llvm_throw_corlib_exception_trampoline", sig);
2009 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2010 * - On x86, LLVM generated code doesn't push the arguments
2011 * - The trampoline takes the throw address as an arguments, not a pc offset.
2013 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
2015 mono_memory_barrier ();
2016 ctx->module->throw_corlib_exception = callee;
2020 if (IS_TARGET_X86 || IS_TARGET_AMD64)
2021 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2023 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE);
2026 * The LLVM mono branch contains changes so a block address can be passed as an
2027 * argument to a call.
2029 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2030 emit_call (ctx, bb, &builder, callee, args, 2);
2032 LLVMBuildUnreachable (builder);
2034 ctx->builder = builder = create_builder (ctx);
2035 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2037 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2044 * emit_args_to_vtype:
2046 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2049 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2051 int j, size, nslots;
2053 size = get_vtype_size (t);
2055 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2056 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2059 if (ainfo->storage == LLVMArgAsFpArgs)
2060 nslots = ainfo->nslots;
2064 for (j = 0; j < nslots; ++j) {
2065 LLVMValueRef index [2], addr, daddr;
2066 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2067 LLVMTypeRef part_type;
2069 if (ainfo->pair_storage [j] == LLVMArgNone)
2072 switch (ainfo->pair_storage [j]) {
2073 case LLVMArgInIReg: {
2074 part_type = LLVMIntType (part_size * 8);
2075 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2076 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2077 addr = LLVMBuildGEP (builder, address, index, 1, "");
2079 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2080 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2081 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2083 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2086 case LLVMArgInFPReg: {
2087 LLVMTypeRef arg_type;
2089 if (ainfo->esize == 8)
2090 arg_type = LLVMDoubleType ();
2092 arg_type = LLVMFloatType ();
2094 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2095 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2096 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2097 LLVMBuildStore (builder, args [j], addr);
2103 g_assert_not_reached ();
2106 size -= sizeof (gpointer);
2111 * emit_vtype_to_args:
2113 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2114 * into ARGS, and the number of arguments into NARGS.
2117 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2120 int j, size, nslots;
2121 LLVMTypeRef arg_type;
2123 size = get_vtype_size (t);
2125 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t)))
2126 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2128 if (ainfo->storage == LLVMArgAsFpArgs)
2129 nslots = ainfo->nslots;
2132 for (j = 0; j < nslots; ++j) {
2133 LLVMValueRef index [2], addr, daddr;
2134 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2136 if (ainfo->pair_storage [j] == LLVMArgNone)
2139 switch (ainfo->pair_storage [j]) {
2141 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2142 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2143 addr = LLVMBuildGEP (builder, address, index, 1, "");
2145 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2146 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2147 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2149 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2151 case LLVMArgInFPReg:
2152 if (ainfo->esize == 8)
2153 arg_type = LLVMDoubleType ();
2155 arg_type = LLVMFloatType ();
2156 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2157 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2158 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2159 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2164 g_assert_not_reached ();
2166 size -= sizeof (gpointer);
2173 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2176 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2177 * get executed every time control reaches them.
2179 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2181 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2182 return ctx->last_alloca;
2186 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2188 return build_alloca_llvm_type_name (ctx, t, align, "");
2192 build_alloca (EmitContext *ctx, MonoType *t)
2194 MonoClass *k = mono_class_from_mono_type (t);
2197 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2200 align = mono_class_min_align (k);
2202 /* Sometimes align is not a power of 2 */
2203 while (mono_is_power_of_two (align) == -1)
2206 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2210 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2213 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2216 module->used = g_ptr_array_sized_new (16);
2217 g_ptr_array_add (module->used, global);
2221 emit_llvm_used (MonoLLVMModule *module)
2223 LLVMModuleRef lmodule = module->lmodule;
2224 LLVMTypeRef used_type;
2225 LLVMValueRef used, *used_elem;
2231 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2232 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2233 used_elem = g_new0 (LLVMValueRef, module->used->len);
2234 for (i = 0; i < module->used->len; ++i)
2235 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2236 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2237 LLVMSetLinkage (used, LLVMAppendingLinkage);
2238 LLVMSetSection (used, "llvm.metadata");
2244 * Emit a function mapping method indexes to their code
2247 emit_get_method (MonoLLVMModule *module)
2249 LLVMModuleRef lmodule = module->lmodule;
2250 LLVMValueRef func, switch_ins, m;
2251 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2252 LLVMBasicBlockRef *bbs;
2254 LLVMBuilderRef builder;
2259 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2260 * but generating code seems safer.
2262 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2263 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2264 LLVMSetLinkage (func, LLVMExternalLinkage);
2265 LLVMSetVisibility (func, LLVMHiddenVisibility);
2266 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2267 module->get_method = func;
2269 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2272 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2273 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2274 * then we will have to find another solution.
2277 name = g_strdup_printf ("BB_CODE_START");
2278 code_start_bb = LLVMAppendBasicBlock (func, name);
2280 builder = LLVMCreateBuilder ();
2281 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2282 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2284 name = g_strdup_printf ("BB_CODE_END");
2285 code_end_bb = LLVMAppendBasicBlock (func, name);
2287 builder = LLVMCreateBuilder ();
2288 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2289 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2291 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2292 for (i = 0; i < module->max_method_idx + 1; ++i) {
2293 name = g_strdup_printf ("BB_%d", i);
2294 bb = LLVMAppendBasicBlock (func, name);
2298 builder = LLVMCreateBuilder ();
2299 LLVMPositionBuilderAtEnd (builder, bb);
2301 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2303 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2305 LLVMBuildRet (builder, LLVMConstNull (rtype));
2308 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2309 builder = LLVMCreateBuilder ();
2310 LLVMPositionBuilderAtEnd (builder, fail_bb);
2311 LLVMBuildRet (builder, LLVMConstNull (rtype));
2313 builder = LLVMCreateBuilder ();
2314 LLVMPositionBuilderAtEnd (builder, entry_bb);
2316 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2317 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2318 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2319 for (i = 0; i < module->max_method_idx + 1; ++i) {
2320 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2323 mark_as_used (module, func);
2327 * emit_get_unbox_tramp:
2329 * Emit a function mapping method indexes to their unbox trampoline
2332 emit_get_unbox_tramp (MonoLLVMModule *module)
2334 LLVMModuleRef lmodule = module->lmodule;
2335 LLVMValueRef func, switch_ins, m;
2336 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2337 LLVMBasicBlockRef *bbs;
2339 LLVMBuilderRef builder;
2343 /* Similar to emit_get_method () */
2345 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2346 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2347 LLVMSetLinkage (func, LLVMExternalLinkage);
2348 LLVMSetVisibility (func, LLVMHiddenVisibility);
2349 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2350 module->get_unbox_tramp = func;
2352 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2354 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2355 for (i = 0; i < module->max_method_idx + 1; ++i) {
2356 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2360 name = g_strdup_printf ("BB_%d", i);
2361 bb = LLVMAppendBasicBlock (func, name);
2365 builder = LLVMCreateBuilder ();
2366 LLVMPositionBuilderAtEnd (builder, bb);
2368 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2371 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2372 builder = LLVMCreateBuilder ();
2373 LLVMPositionBuilderAtEnd (builder, fail_bb);
2374 LLVMBuildRet (builder, LLVMConstNull (rtype));
2376 builder = LLVMCreateBuilder ();
2377 LLVMPositionBuilderAtEnd (builder, entry_bb);
2379 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2380 for (i = 0; i < module->max_method_idx + 1; ++i) {
2381 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2385 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2388 mark_as_used (module, func);
2391 /* Add a function to mark the beginning of LLVM code */
2393 emit_llvm_code_start (MonoLLVMModule *module)
2395 LLVMModuleRef lmodule = module->lmodule;
2397 LLVMBasicBlockRef entry_bb;
2398 LLVMBuilderRef builder;
2400 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2401 LLVMSetLinkage (func, LLVMInternalLinkage);
2402 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2403 module->code_start = func;
2404 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2405 builder = LLVMCreateBuilder ();
2406 LLVMPositionBuilderAtEnd (builder, entry_bb);
2407 LLVMBuildRetVoid (builder);
2411 emit_init_icall_wrapper (MonoLLVMModule *module, const char *name, const char *icall_name, int subtype)
2413 LLVMModuleRef lmodule = module->lmodule;
2414 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2415 LLVMBasicBlockRef entry_bb;
2416 LLVMBuilderRef builder;
2423 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
2424 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
2427 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
2428 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
2431 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
2432 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
2435 g_assert_not_reached ();
2437 LLVMSetLinkage (func, LLVMInternalLinkage);
2438 LLVMAddFunctionAttr (func, LLVMNoInlineAttribute);
2439 mono_llvm_set_preserveall_cc (func);
2440 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2441 builder = LLVMCreateBuilder ();
2442 LLVMPositionBuilderAtEnd (builder, entry_bb);
2445 ji = g_new0 (MonoJumpInfo, 1);
2446 ji->type = MONO_PATCH_INFO_AOT_MODULE;
2447 ji = mono_aot_patch_info_dup (ji);
2448 got_offset = mono_aot_get_got_offset (ji);
2449 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2450 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2451 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2452 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2453 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
2454 args [1] = LLVMGetParam (func, 0);
2456 args [2] = LLVMGetParam (func, 1);
2458 ji = g_new0 (MonoJumpInfo, 1);
2459 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
2460 ji->data.name = icall_name;
2461 ji = mono_aot_patch_info_dup (ji);
2462 got_offset = mono_aot_get_got_offset (ji);
2463 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2464 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2465 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2466 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2467 callee = LLVMBuildLoad (builder, got_entry_addr, "");
2468 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
2469 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
2471 // Set the inited flag
2472 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2473 indexes [1] = LLVMGetParam (func, 0);
2474 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
2476 LLVMBuildRetVoid (builder);
2478 LLVMVerifyFunction(func, 0);
2483 * Emit wrappers around the C icalls used to initialize llvm methods, to
2484 * make the calling code smaller and to enable usage of the llvm
2485 * PreserveAll calling convention.
2488 emit_init_icall_wrappers (MonoLLVMModule *module)
2490 module->init_method = emit_init_icall_wrapper (module, "init_method", "mono_aot_init_llvm_method", 0);
2491 module->init_method_gshared_rgctx = emit_init_icall_wrapper (module, "init_method_gshared_rgctx", "mono_aot_init_gshared_method_rgctx", 1);
2492 module->init_method_gshared_this = emit_init_icall_wrapper (module, "init_method_gshared_this", "mono_aot_init_gshared_method_this", 2);
2496 emit_llvm_code_end (MonoLLVMModule *module)
2498 LLVMModuleRef lmodule = module->lmodule;
2500 LLVMBasicBlockRef entry_bb;
2501 LLVMBuilderRef builder;
2503 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2504 LLVMSetLinkage (func, LLVMInternalLinkage);
2505 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2506 module->code_end = func;
2507 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2508 builder = LLVMCreateBuilder ();
2509 LLVMPositionBuilderAtEnd (builder, entry_bb);
2510 LLVMBuildRetVoid (builder);
2514 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
2516 gboolean need_div_check = ctx->cfg->backend->need_div_check;
2519 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
2520 need_div_check = TRUE;
2522 if (!need_div_check)
2525 switch (ins->opcode) {
2538 case OP_IDIV_UN_IMM:
2539 case OP_LDIV_UN_IMM:
2540 case OP_IREM_UN_IMM:
2541 case OP_LREM_UN_IMM: {
2543 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
2544 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
2546 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
2547 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
2548 CHECK_FAILURE (ctx);
2549 builder = ctx->builder;
2551 /* b == -1 && a == 0x80000000 */
2553 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
2554 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
2555 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
2557 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
2558 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
2559 CHECK_FAILURE (ctx);
2560 builder = ctx->builder;
2575 * Emit code to initialize the GOT slots used by the method.
2578 emit_init_method (EmitContext *ctx)
2580 LLVMValueRef indexes [16], args [16], callee;
2581 LLVMValueRef inited_var, cmp, call;
2582 LLVMBasicBlockRef inited_bb, notinited_bb;
2583 LLVMBuilderRef builder = ctx->builder;
2584 MonoCompile *cfg = ctx->cfg;
2586 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
2587 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
2589 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2590 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
2591 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
2593 args [0] = inited_var;
2594 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
2595 inited_var = LLVMBuildCall (ctx->builder, LLVMGetNamedFunction (ctx->lmodule, "llvm.expect.i8"), args, 2, "");
2597 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
2599 inited_bb = ctx->inited_bb;
2600 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
2602 LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
2604 builder = ctx->builder = create_builder (ctx);
2605 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
2608 if (ctx->rgctx_arg) {
2609 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2610 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2611 callee = ctx->module->init_method_gshared_rgctx;
2612 call = LLVMBuildCall (builder, callee, args, 2, "");
2613 } else if (cfg->gshared) {
2614 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2615 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
2616 callee = ctx->module->init_method_gshared_this;
2617 call = LLVMBuildCall (builder, callee, args, 2, "");
2619 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2620 callee = ctx->module->init_method;
2621 call = LLVMBuildCall (builder, callee, args, 1, "");
2625 * This enables llvm to keep arguments in their original registers/
2626 * scratch registers, since the call will not clobber them.
2628 mono_llvm_set_call_preserveall_cc (call);
2630 LLVMBuildBr (builder, inited_bb);
2631 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
2633 builder = ctx->builder = create_builder (ctx);
2634 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
2638 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
2641 * Emit unbox trampoline using a tail call
2643 LLVMValueRef tramp, call, *args;
2644 LLVMBuilderRef builder;
2645 LLVMBasicBlockRef lbb;
2649 tramp_name = g_strdup_printf ("ut_%s", method_name);
2650 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
2651 LLVMSetLinkage (tramp, LLVMInternalLinkage);
2652 LLVMAddFunctionAttr (tramp, LLVMNoUnwindAttribute);
2653 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2654 LLVMAddAttribute (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVMInRegAttribute);
2656 lbb = LLVMAppendBasicBlock (tramp, "");
2657 builder = LLVMCreateBuilder ();
2658 LLVMPositionBuilderAtEnd (builder, lbb);
2660 nargs = LLVMCountParamTypes (method_type);
2661 args = g_new0 (LLVMValueRef, nargs);
2662 for (i = 0; i < nargs; ++i) {
2663 args [i] = LLVMGetParam (tramp, i);
2664 if (i == ctx->this_arg_pindex) {
2665 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
2667 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
2668 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), sizeof (MonoObject), FALSE), "");
2669 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
2672 call = LLVMBuildCall (builder, method, args, nargs, "");
2673 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2674 LLVMAddInstrAttribute (call, 1 + ctx->rgctx_arg_pindex, LLVMInRegAttribute);
2675 mono_llvm_set_must_tail (call);
2676 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
2677 LLVMBuildRetVoid (builder);
2679 LLVMBuildRet (builder, call);
2681 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
2687 * Emit code to load/convert arguments.
2690 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
2693 MonoCompile *cfg = ctx->cfg;
2694 MonoMethodSignature *sig = ctx->sig;
2695 LLVMCallInfo *linfo = ctx->linfo;
2699 LLVMBuilderRef old_builder = ctx->builder;
2700 ctx->builder = builder;
2702 ctx->alloca_builder = create_builder (ctx);
2705 * Handle indirect/volatile variables by allocating memory for them
2706 * using 'alloca', and storing their address in a temporary.
2708 for (i = 0; i < cfg->num_varinfo; ++i) {
2709 MonoInst *var = cfg->varinfo [i];
2712 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
2713 vtype = type_to_llvm_type (ctx, var->inst_vtype);
2714 CHECK_FAILURE (ctx);
2715 /* Could be already created by an OP_VPHI */
2716 if (!ctx->addresses [var->dreg])
2717 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
2718 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
2722 names = g_new (char *, sig->param_count);
2723 mono_method_get_param_names (cfg->method, (const char **) names);
2725 for (i = 0; i < sig->param_count; ++i) {
2726 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
2727 int reg = cfg->args [i + sig->hasthis]->dreg;
2730 pindex = ainfo->pindex;
2732 switch (ainfo->storage) {
2733 case LLVMArgVtypeInReg:
2734 case LLVMArgAsFpArgs: {
2735 LLVMValueRef args [8];
2738 /* The argument is received as a set of int/fp arguments, store them into the real argument */
2739 memset (args, 0, sizeof (args));
2740 if (ainfo->storage == LLVMArgVtypeInReg) {
2741 args [0] = LLVMGetParam (ctx->lmethod, pindex);
2742 if (ainfo->pair_storage [1] != LLVMArgNone)
2743 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
2745 g_assert (ainfo->nslots <= 8);
2746 for (j = 0; j < ainfo->nslots; ++j)
2747 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
2749 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2751 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
2753 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2754 /* Treat these as normal values */
2755 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2759 case LLVMArgVtypeByVal: {
2760 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2762 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2763 /* Treat these as normal values */
2764 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2768 case LLVMArgVtypeByRef: {
2769 /* The argument is passed by ref */
2770 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2773 case LLVMArgScalarByRef: {
2775 name = g_strdup_printf ("arg_%s", names [i]);
2777 name = g_strdup_printf ("arg_%d", i);
2778 ctx->values [reg] = LLVMBuildLoad (builder, LLVMGetParam (ctx->lmethod, pindex), name);
2782 case LLVMArgAsIArgs: {
2783 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2785 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2787 /* The argument is received as an array of ints, store it into the real argument */
2788 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
2791 case LLVMArgVtypeAsScalar:
2792 g_assert_not_reached ();
2795 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
2802 emit_volatile_store (ctx, cfg->vret_addr->dreg);
2804 emit_volatile_store (ctx, cfg->args [0]->dreg);
2805 for (i = 0; i < sig->param_count; ++i)
2806 if (!mini_type_is_vtype (sig->params [i]))
2807 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
2809 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
2810 LLVMValueRef this_alloc;
2813 * The exception handling code needs the location where the this argument was
2814 * stored for gshared methods. We create a separate alloca to hold it, and mark it
2815 * with the "mono.this" custom metadata to tell llvm that it needs to save its
2816 * location into the LSDA.
2818 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
2819 /* This volatile store will keep the alloca alive */
2820 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
2822 set_metadata_flag (this_alloc, "mono.this");
2825 if (cfg->rgctx_var) {
2826 LLVMValueRef rgctx_alloc, store;
2829 * We handle the rgctx arg similarly to the this pointer.
2831 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
2832 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
2833 /* This volatile store will keep the alloca alive */
2834 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
2836 set_metadata_flag (rgctx_alloc, "mono.this");
2839 /* Initialize the method if needed */
2840 if (cfg->compile_aot && ctx->llvm_only) {
2841 /* Emit a location for the initialization code */
2842 ctx->init_bb = gen_bb (ctx, "INIT_BB");
2843 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
2845 LLVMBuildBr (ctx->builder, ctx->init_bb);
2846 builder = ctx->builder = create_builder (ctx);
2847 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
2848 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
2851 /* Compute nesting between clauses */
2852 ctx->nested_in = mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
2853 for (i = 0; i < cfg->header->num_clauses; ++i) {
2854 for (j = 0; j < cfg->header->num_clauses; ++j) {
2855 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
2856 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
2858 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
2859 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
2864 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
2865 * it needs to continue normally, or return back to the exception handling system.
2867 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
2871 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
2874 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
2875 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
2876 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
2878 if (bb->in_scount == 0) {
2881 sprintf (name, "finally_ind_bb%d", bb->block_num);
2882 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
2883 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
2885 ctx->bblocks [bb->block_num].finally_ind = val;
2887 /* Create a variable to hold the exception var */
2889 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
2893 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
2894 * LLVM bblock containing a landing pad causes problems for the
2895 * LLVM optimizer passes.
2897 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
2898 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
2905 ctx->builder = old_builder;
2909 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
2911 MonoCompile *cfg = ctx->cfg;
2912 LLVMModuleRef lmodule = ctx->lmodule;
2913 LLVMValueRef *values = ctx->values;
2914 LLVMValueRef *addresses = ctx->addresses;
2915 MonoCallInst *call = (MonoCallInst*)ins;
2916 MonoMethodSignature *sig = call->signature;
2917 LLVMValueRef callee = NULL, lcall;
2919 LLVMCallInfo *cinfo;
2923 LLVMTypeRef llvm_sig;
2925 gboolean is_virtual, calli;
2926 LLVMBuilderRef builder = *builder_ref;
2928 if (call->signature->call_convention != MONO_CALL_DEFAULT)
2929 LLVM_FAILURE (ctx, "non-default callconv");
2931 cinfo = call->cinfo;
2933 if (call->rgctx_arg_reg)
2934 cinfo->rgctx_arg = TRUE;
2935 if (call->imt_arg_reg)
2936 cinfo->imt_arg = TRUE;
2938 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgScalarRetAddr);
2940 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
2941 CHECK_FAILURE (ctx);
2943 is_virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE || ins->opcode == OP_RCALL_MEMBASE);
2944 calli = !call->fptr_is_patch && (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG || ins->opcode == OP_RCALL_REG);
2946 /* FIXME: Avoid creating duplicate methods */
2948 if (ins->flags & MONO_INST_HAS_METHOD) {
2952 if (cfg->compile_aot) {
2953 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
2955 LLVM_FAILURE (ctx, "can't encode patch");
2957 if (cfg->llvm_only && call->method->klass->image->assembly == ctx->module->assembly) {
2959 * Collect instructions representing the callee into a hash so they can be replaced
2960 * by the llvm method for the callee if the callee turns out to be direct
2961 * callable. Currently this only requires it to not fail llvm compilation.
2963 GSList *l = g_hash_table_lookup (ctx->method_to_callers, call->method);
2964 l = g_slist_prepend (l, callee);
2965 g_hash_table_insert (ctx->method_to_callers, call->method, l);
2968 callee = LLVMAddFunction (lmodule, "", llvm_sig);
2971 mono_create_jit_trampoline_in_domain (mono_domain_get (),
2973 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
2977 if (call->method && strstr (call->method->klass->name, "AsyncVoidMethodBuilder"))
2978 /* LLVM miscompiles async methods */
2979 LLVM_FAILURE (ctx, "#13734");
2982 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
2988 memset (&ji, 0, sizeof (ji));
2989 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
2990 ji.data.target = info->name;
2992 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
2994 if (cfg->compile_aot) {
2995 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
2997 LLVM_FAILURE (ctx, "can't encode patch");
2999 callee = LLVMAddFunction (lmodule, "", llvm_sig);
3000 target = (gpointer)mono_icall_get_wrapper (info);
3001 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3004 if (cfg->compile_aot) {
3006 if (cfg->abs_patches) {
3007 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3009 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3011 LLVM_FAILURE (ctx, "can't encode patch");
3015 LLVM_FAILURE (ctx, "aot");
3017 callee = LLVMAddFunction (lmodule, "", llvm_sig);
3019 if (cfg->abs_patches) {
3020 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3023 * FIXME: Some trampolines might have
3024 * their own calling convention on some platforms.
3026 #ifndef TARGET_AMD64
3027 if (abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER || abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER_V4 ||
3028 abs_ji->type == MONO_PATCH_INFO_MONITOR_EXIT)
3029 LLVM_FAILURE (ctx, "trampoline with own cconv");
3031 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE);
3032 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3036 LLVMAddGlobalMapping (ctx->module->ee, callee, (gpointer)call->fptr);
3042 int size = sizeof (gpointer);
3045 g_assert (ins->inst_offset % size == 0);
3046 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3048 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3050 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3052 if (ins->flags & MONO_INST_HAS_METHOD) {
3057 * Collect and convert arguments
3059 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg;
3060 len = sizeof (LLVMValueRef) * nargs;
3061 args = (LLVMValueRef*)alloca (len);
3062 memset (args, 0, len);
3063 l = call->out_ireg_args;
3065 if (call->rgctx_arg_reg) {
3066 g_assert (values [call->rgctx_arg_reg]);
3067 g_assert (cinfo->rgctx_arg_pindex < nargs);
3069 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3070 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3071 * it using a volatile load.
3074 if (!ctx->imt_rgctx_loc)
3075 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3076 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3077 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
3079 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3082 if (call->imt_arg_reg) {
3083 g_assert (!ctx->llvm_only);
3084 g_assert (values [call->imt_arg_reg]);
3085 g_assert (cinfo->imt_arg_pindex < nargs);
3087 if (!ctx->imt_rgctx_loc)
3088 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3089 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3090 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
3092 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3096 if (!addresses [call->inst.dreg])
3097 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3098 g_assert (cinfo->vret_arg_pindex < nargs);
3099 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3100 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3102 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
3105 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3108 LLVMArgInfo *ainfo = &call->cinfo->args [i];
3110 pindex = ainfo->pindex;
3112 regpair = (guint32)(gssize)(l->data);
3113 reg = regpair & 0xffffff;
3114 args [pindex] = values [reg];
3115 switch (ainfo->storage) {
3116 case LLVMArgVtypeInReg:
3117 case LLVMArgAsFpArgs: {
3120 g_assert (addresses [reg]);
3121 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
3125 // FIXME: Get rid of the VMOVE
3128 case LLVMArgVtypeByVal:
3129 g_assert (addresses [reg]);
3130 args [pindex] = addresses [reg];
3132 case LLVMArgVtypeByRef:
3133 case LLVMArgScalarByRef: {
3134 g_assert (addresses [reg]);
3135 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3138 case LLVMArgAsIArgs:
3139 g_assert (addresses [reg]);
3140 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
3142 case LLVMArgVtypeAsScalar:
3143 g_assert_not_reached ();
3146 g_assert (args [pindex]);
3147 if (i == 0 && sig->hasthis)
3148 args [pindex] = convert (ctx, args [pindex], ThisType ());
3150 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
3153 g_assert (pindex <= nargs);
3158 // FIXME: Align call sites
3164 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
3167 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
3169 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
3170 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
3172 /* The two can't be used together, so use only one LLVM calling conv to pass them */
3173 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
3174 if (!sig->pinvoke && !cfg->llvm_only)
3175 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
3177 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3178 LLVMAddInstrAttribute (lcall, 1 + cinfo->vret_arg_pindex, LLVMStructRetAttribute);
3179 if (!ctx->llvm_only && call->rgctx_arg_reg)
3180 LLVMAddInstrAttribute (lcall, 1 + cinfo->rgctx_arg_pindex, LLVMInRegAttribute);
3181 if (call->imt_arg_reg)
3182 LLVMAddInstrAttribute (lcall, 1 + cinfo->imt_arg_pindex, LLVMInRegAttribute);
3184 /* Add byval attributes if needed */
3185 for (i = 0; i < sig->param_count; ++i) {
3186 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
3188 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
3189 LLVMAddInstrAttribute (lcall, 1 + ainfo->pindex, LLVMByValAttribute);
3193 * Convert the result
3195 switch (cinfo->ret.storage) {
3196 case LLVMArgVtypeInReg: {
3197 LLVMValueRef regs [2];
3199 if (LLVMTypeOf (lcall) == LLVMVoidType ())
3203 if (!addresses [ins->dreg])
3204 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
3206 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
3207 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
3208 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
3209 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
3212 case LLVMArgVtypeByVal:
3213 if (!addresses [call->inst.dreg])
3214 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3215 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
3217 case LLVMArgFpStruct:
3218 if (!addresses [call->inst.dreg])
3219 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3220 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3222 case LLVMArgVtypeAsScalar:
3223 if (!addresses [call->inst.dreg])
3224 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3225 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3227 case LLVMArgVtypeRetAddr:
3228 case LLVMArgVtypeByRef:
3230 case LLVMArgScalarRetAddr:
3231 /* Normal scalar returned using a vtype return argument */
3232 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3235 if (sig->ret->type != MONO_TYPE_VOID)
3236 /* If the method returns an unsigned value, need to zext it */
3237 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
3241 *builder_ref = ctx->builder;
3249 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3251 const char *icall_name = rethrow ? "mono_llvm_rethrow_exception" : "mono_llvm_throw_exception";
3252 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3254 LLVMTypeRef exc_type = type_to_llvm_type (ctx, &mono_get_exception_class ()->byval_arg);
3257 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
3259 if (ctx->cfg->compile_aot) {
3260 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3262 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3263 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3264 mono_memory_barrier ();
3267 ctx->module->rethrow = callee;
3269 ctx->module->throw_icall = callee;
3273 LLVMValueRef args [2];
3275 args [0] = convert (ctx, exc, exc_type);
3276 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
3278 LLVMBuildUnreachable (ctx->builder);
3280 ctx->builder = create_builder (ctx);
3284 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3286 MonoMethodSignature *throw_sig;
3287 LLVMValueRef callee, arg;
3288 const char *icall_name;
3290 callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3291 icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
3294 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
3295 throw_sig->ret = &mono_get_void_class ()->byval_arg;
3296 throw_sig->params [0] = &mono_get_object_class ()->byval_arg;
3297 if (ctx->cfg->compile_aot) {
3298 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3300 callee = LLVMAddFunction (ctx->lmodule, icall_name, sig_to_llvm_sig (ctx, throw_sig));
3304 * LLVM doesn't push the exception argument, so we need a different
3307 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline"));
3309 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3313 mono_memory_barrier ();
3315 ctx->module->rethrow = callee;
3317 ctx->module->throw_icall = callee;
3319 arg = convert (ctx, exc, type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg));
3320 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
3324 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
3326 const char *icall_name = "mono_llvm_resume_exception";
3327 LLVMValueRef callee = ctx->module->resume_eh;
3329 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3332 if (ctx->cfg->compile_aot) {
3333 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3335 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3336 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3337 mono_memory_barrier ();
3339 ctx->module->resume_eh = callee;
3343 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
3345 LLVMBuildUnreachable (ctx->builder);
3347 ctx->builder = create_builder (ctx);
3351 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3353 const char *icall_name = "mono_llvm_clear_exception";
3355 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3356 LLVMValueRef callee = NULL;
3359 if (ctx->cfg->compile_aot) {
3360 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3362 // FIXME: This is broken.
3363 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3367 g_assert (builder && callee);
3369 return LLVMBuildCall (builder, callee, NULL, 0, "");
3373 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3375 const char *icall_name = "mono_llvm_load_exception";
3377 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
3378 LLVMValueRef callee = NULL;
3381 if (ctx->cfg->compile_aot) {
3382 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3384 // FIXME: This is broken.
3385 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3389 g_assert (builder && callee);
3391 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
3396 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
3398 const char *icall_name = "mono_llvm_match_exception";
3400 ctx->builder = builder;
3402 const int num_args = 3;
3403 LLVMValueRef args [num_args];
3404 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
3405 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
3406 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
3408 LLVMTypeRef match_sig = LLVMFunctionType3 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), FALSE);
3409 LLVMValueRef callee = ctx->module->match_exc;
3412 if (ctx->cfg->compile_aot) {
3413 ctx->builder = builder;
3414 // get_callee expects ctx->builder to be the emitting builder
3415 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3417 callee = ctx->module->match_exc = LLVMAddFunction (ctx->lmodule, icall_name, match_sig);
3418 LLVMAddGlobalMapping (ctx->module->ee, ctx->module->match_exc, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3419 ctx->module->match_exc = callee;
3420 mono_memory_barrier ();
3424 g_assert (builder && callee);
3426 g_assert (ctx->ex_var);
3428 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
3431 // FIXME: This won't work because the code-finding makes this
3433 /*#define MONO_PERSONALITY_DEBUG*/
3435 #ifdef MONO_PERSONALITY_DEBUG
3436 static const gboolean use_debug_personality = TRUE;
3437 static const char *default_personality_name = "mono_debug_personality";
3439 static const gboolean use_debug_personality = FALSE;
3440 static const char *default_personality_name = "__gxx_personality_v0";
3444 default_cpp_lpad_exc_signature (void)
3446 static gboolean inited = FALSE;
3447 static LLVMTypeRef sig;
3450 LLVMTypeRef signature [2];
3451 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
3452 signature [1] = LLVMInt32Type ();
3453 sig = LLVMStructType (signature, 2, FALSE);
3461 get_mono_personality (EmitContext *ctx)
3463 LLVMValueRef personality = NULL;
3464 static gint32 mapping_inited = FALSE;
3465 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
3467 if (!use_debug_personality) {
3468 if (ctx->cfg->compile_aot) {
3469 personality = LLVMGetNamedFunction (ctx->lmodule, default_personality_name);
3470 } else if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0) {
3471 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3472 LLVMAddGlobalMapping (ctx->module->ee, personality, personality);
3475 if (ctx->cfg->compile_aot) {
3476 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name);
3478 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3479 LLVMAddGlobalMapping (ctx->module->ee, personality, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name));
3480 mono_memory_barrier ();
3484 g_assert (personality);
3488 static LLVMBasicBlockRef
3489 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
3491 MonoCompile *cfg = ctx->cfg;
3492 LLVMBuilderRef old_builder = ctx->builder;
3493 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
3495 LLVMBuilderRef lpadBuilder = create_builder (ctx);
3496 ctx->builder = lpadBuilder;
3498 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
3499 g_assert (handler_bb);
3501 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3502 LLVMValueRef personality = get_mono_personality (ctx);
3503 g_assert (personality);
3505 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
3506 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
3508 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3509 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
3510 g_assert (landing_pad);
3512 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
3513 LLVMAddClause (landing_pad, cast);
3515 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
3516 LLVMBuilderRef resume_builder = create_builder (ctx);
3517 ctx->builder = resume_builder;
3518 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
3520 emit_resume_eh (ctx, handler_bb);
3523 ctx->builder = lpadBuilder;
3524 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3526 gboolean finally_only = TRUE;
3528 MonoExceptionClause *group_cursor = group_start;
3530 for (int i = 0; i < group_size; i ++) {
3531 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY))
3532 finally_only = FALSE;
3538 // Handle landing pad inlining
3540 if (!finally_only) {
3541 // So at each level of the exception stack we will match the exception again.
3542 // During that match, we need to compare against the handler types for the current
3543 // protected region. We send the try start and end so that we can only check against
3544 // handlers for this lexical protected region.
3545 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
3547 // if returns -1, resume
3548 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
3550 // else move to that target bb
3551 for (int i=0; i < group_size; i++) {
3552 MonoExceptionClause *clause = group_start + i;
3553 int clause_index = clause - cfg->header->clauses;
3554 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3555 g_assert (handler_bb);
3556 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3557 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3560 int clause_index = group_start - cfg->header->clauses;
3561 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3562 g_assert (finally_bb);
3564 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
3567 ctx->builder = old_builder;
3574 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
3576 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3577 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
3579 // Make exception available to catch blocks
3580 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY)) {
3581 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
3583 g_assert (ctx->ex_var);
3584 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
3586 if (bb->in_scount == 1) {
3587 MonoInst *exvar = bb->in_stack [0];
3588 g_assert (!ctx->values [exvar->dreg]);
3589 g_assert (ctx->ex_var);
3590 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
3591 emit_volatile_store (ctx, exvar->dreg);
3594 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
3597 LLVMBuilderRef handler_builder = create_builder (ctx);
3598 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
3599 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
3601 // Make the handler code end with a jump to cbb
3602 LLVMBuildBr (handler_builder, cbb);
3606 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
3608 MonoCompile *cfg = ctx->cfg;
3609 LLVMValueRef *values = ctx->values;
3610 LLVMModuleRef lmodule = ctx->lmodule;
3611 BBInfo *bblocks = ctx->bblocks;
3613 LLVMValueRef personality;
3614 LLVMValueRef landing_pad;
3615 LLVMBasicBlockRef target_bb;
3617 static gint32 mapping_inited;
3618 static int ti_generator;
3621 LLVMValueRef type_info;
3625 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3627 if (cfg->compile_aot) {
3628 /* Use a dummy personality function */
3629 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3630 g_assert (personality);
3632 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3633 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
3634 LLVMAddGlobalMapping (ctx->module->ee, personality, mono_personality);
3637 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
3639 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
3642 * Create the type info
3644 sprintf (ti_name, "type_info_%d", ti_generator);
3647 if (cfg->compile_aot) {
3648 /* decode_eh_frame () in aot-runtime.c will decode this */
3649 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
3650 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
3653 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
3655 LLVMSetLinkage (type_info, LLVMInternalLinkage);
3658 * After the cfg mempool is freed, the type info will point to stale memory,
3659 * but this is not a problem, since we decode it once in exception_cb during
3662 ti = mono_mempool_alloc (cfg->mempool, sizeof (gint32));
3663 *(gint32*)ti = clause_index;
3665 type_info = LLVMAddGlobal (lmodule, i8ptr, ti_name);
3667 LLVMAddGlobalMapping (ctx->module->ee, type_info, ti);
3671 LLVMTypeRef members [2], ret_type;
3673 members [0] = i8ptr;
3674 members [1] = LLVMInt32Type ();
3675 ret_type = LLVMStructType (members, 2, FALSE);
3677 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
3678 LLVMAddClause (landing_pad, type_info);
3680 /* Store the exception into the exvar */
3682 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
3686 * LLVM throw sites are associated with a one landing pad, and LLVM generated
3687 * code expects control to be transferred to this landing pad even in the
3688 * presence of nested clauses. The landing pad needs to branch to the landing
3689 * pads belonging to nested clauses based on the selector value returned by
3690 * the landing pad instruction, which is passed to the landing pad in a
3691 * register by the EH code.
3693 target_bb = bblocks [bb->block_num].call_handler_target_bb;
3694 g_assert (target_bb);
3697 * Branch to the correct landing pad
3699 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
3700 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
3702 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
3703 int nesting_clause_index = GPOINTER_TO_INT (l->data);
3704 MonoBasicBlock *handler_bb;
3706 handler_bb = g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
3707 g_assert (handler_bb);
3709 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3710 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3713 /* Start a new bblock which CALL_HANDLER can branch to */
3714 target_bb = bblocks [bb->block_num].call_handler_target_bb;
3716 ctx->builder = builder = create_builder (ctx);
3717 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
3719 ctx->bblocks [bb->block_num].end_bblock = target_bb;
3721 /* Store the exception into the IL level exvar */
3722 if (bb->in_scount == 1) {
3723 g_assert (bb->in_scount == 1);
3724 exvar = bb->in_stack [0];
3726 // FIXME: This is shared with filter clauses ?
3727 g_assert (!values [exvar->dreg]);
3729 g_assert (ctx->ex_var);
3730 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
3731 emit_volatile_store (ctx, exvar->dreg);
3737 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
3739 MonoCompile *cfg = ctx->cfg;
3740 MonoMethodSignature *sig = ctx->sig;
3741 LLVMValueRef method = ctx->lmethod;
3742 LLVMValueRef *values = ctx->values;
3743 LLVMValueRef *addresses = ctx->addresses;
3744 LLVMCallInfo *linfo = ctx->linfo;
3745 LLVMModuleRef lmodule = ctx->lmodule;
3746 BBInfo *bblocks = ctx->bblocks;
3748 LLVMBasicBlockRef cbb;
3749 LLVMBuilderRef builder, starting_builder;
3750 gboolean has_terminator;
3752 LLVMValueRef lhs, rhs;
3755 cbb = get_end_bb (ctx, bb);
3757 builder = create_builder (ctx);
3758 ctx->builder = builder;
3759 LLVMPositionBuilderAtEnd (builder, cbb);
3761 CHECK_FAILURE (ctx);
3763 if (bb->flags & BB_EXCEPTION_HANDLER) {
3764 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
3765 LLVM_FAILURE (ctx, "handler without invokes");
3769 emit_llvmonly_handler_start (ctx, bb, cbb);
3771 emit_handler_start (ctx, bb, builder);
3772 CHECK_FAILURE (ctx);
3773 builder = ctx->builder;
3776 has_terminator = FALSE;
3777 starting_builder = builder;
3778 for (ins = bb->code; ins; ins = ins->next) {
3779 const char *spec = LLVM_INS_INFO (ins->opcode);
3781 char dname_buf [128];
3783 emit_dbg_loc (ctx, builder, ins->cil_code);
3786 if (nins > 3000 && builder == starting_builder) {
3787 /* some steps in llc are non-linear in the size of basic blocks, see #5714 */
3788 LLVM_FAILURE (ctx, "basic block too long");
3792 /* There could be instructions after a terminator, skip them */
3795 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
3796 sprintf (dname_buf, "t%d", ins->dreg);
3800 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
3801 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
3803 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
3804 lhs = emit_volatile_load (ctx, ins->sreg1);
3806 /* It is ok for SETRET to have an uninitialized argument */
3807 if (!values [ins->sreg1] && ins->opcode != OP_SETRET)
3808 LLVM_FAILURE (ctx, "sreg1");
3809 lhs = values [ins->sreg1];
3815 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
3816 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
3817 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
3818 rhs = emit_volatile_load (ctx, ins->sreg2);
3820 if (!values [ins->sreg2])
3821 LLVM_FAILURE (ctx, "sreg2");
3822 rhs = values [ins->sreg2];
3828 //mono_print_ins (ins);
3829 switch (ins->opcode) {
3832 case OP_LIVERANGE_START:
3833 case OP_LIVERANGE_END:
3836 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
3839 #if SIZEOF_VOID_P == 4
3840 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
3842 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
3846 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
3850 values [ins->dreg] = LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0);
3852 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
3854 case OP_DUMMY_ICONST:
3855 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3857 case OP_DUMMY_I8CONST:
3858 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
3860 case OP_DUMMY_R8CONST:
3861 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
3864 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
3865 LLVMBuildBr (builder, target_bb);
3866 has_terminator = TRUE;
3873 LLVMBasicBlockRef new_bb;
3874 LLVMBuilderRef new_builder;
3876 // The default branch is already handled
3877 // FIXME: Handle it here
3879 /* Start new bblock */
3880 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
3881 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
3883 lhs = convert (ctx, lhs, LLVMInt32Type ());
3884 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
3885 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
3886 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
3888 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
3891 new_builder = create_builder (ctx);
3892 LLVMPositionBuilderAtEnd (new_builder, new_bb);
3893 LLVMBuildUnreachable (new_builder);
3895 has_terminator = TRUE;
3896 g_assert (!ins->next);
3902 switch (linfo->ret.storage) {
3903 case LLVMArgVtypeInReg: {
3904 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
3905 LLVMValueRef val, addr, retval;
3908 retval = LLVMGetUndef (ret_type);
3910 if (!addresses [ins->sreg1]) {
3912 * The return type is an LLVM vector type, have to convert between it and the
3913 * real return type which is a struct type.
3915 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret)));
3916 /* Convert to 2xi64 first */
3917 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
3919 for (i = 0; i < 2; ++i) {
3920 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
3921 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
3923 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
3927 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
3928 for (i = 0; i < 2; ++i) {
3929 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
3930 LLVMValueRef indexes [2], part_addr;
3932 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3933 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
3934 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
3936 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
3938 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
3942 LLVMBuildRet (builder, retval);
3945 case LLVMArgVtypeAsScalar: {
3946 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
3947 LLVMValueRef retval;
3950 size = get_vtype_size (sig->ret);
3952 g_assert (addresses [ins->sreg1]);
3954 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
3955 LLVMBuildRet (builder, retval);
3958 case LLVMArgVtypeByVal: {
3959 LLVMValueRef retval;
3961 g_assert (addresses [ins->sreg1]);
3962 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
3963 LLVMBuildRet (builder, retval);
3966 case LLVMArgVtypeByRef: {
3967 LLVMBuildRetVoid (builder);
3970 case LLVMArgVtypeRetAddr: {
3971 LLVMBuildRetVoid (builder);
3974 case LLVMArgScalarRetAddr: {
3975 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
3976 LLVMValueRef param = LLVMGetParam (method, linfo->vret_arg_pindex);
3978 /* sreg1 might not be set */
3980 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, param, LLVMPointerType (ret_type, 0)));
3981 LLVMBuildRetVoid (builder);
3984 case LLVMArgFpStruct: {
3985 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
3986 LLVMValueRef retval;
3988 g_assert (addresses [ins->sreg1]);
3989 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
3990 LLVMBuildRet (builder, retval);
3994 case LLVMArgNormal: {
3995 if (!lhs || ctx->is_dead [ins->sreg1]) {
3997 * The method did not set its return value, probably because it
3998 * ends with a throw.
4001 LLVMBuildRetVoid (builder);
4003 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
4005 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
4007 has_terminator = TRUE;
4011 g_assert_not_reached ();
4020 case OP_ICOMPARE_IMM:
4021 case OP_LCOMPARE_IMM:
4022 case OP_COMPARE_IMM: {
4026 if (ins->next->opcode == OP_NOP)
4029 if (ins->next->opcode == OP_BR)
4030 /* The comparison result is not needed */
4033 rel = mono_opcode_to_cond (ins->next->opcode);
4035 if (ins->opcode == OP_ICOMPARE_IMM) {
4036 lhs = convert (ctx, lhs, LLVMInt32Type ());
4037 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4039 if (ins->opcode == OP_LCOMPARE_IMM) {
4040 lhs = convert (ctx, lhs, LLVMInt64Type ());
4041 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4043 if (ins->opcode == OP_LCOMPARE) {
4044 lhs = convert (ctx, lhs, LLVMInt64Type ());
4045 rhs = convert (ctx, rhs, LLVMInt64Type ());
4047 if (ins->opcode == OP_ICOMPARE) {
4048 lhs = convert (ctx, lhs, LLVMInt32Type ());
4049 rhs = convert (ctx, rhs, LLVMInt32Type ());
4053 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4054 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
4055 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
4056 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
4059 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
4060 if (ins->opcode == OP_FCOMPARE) {
4061 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4062 } else if (ins->opcode == OP_RCOMPARE) {
4063 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4064 } else if (ins->opcode == OP_COMPARE_IMM) {
4065 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0)
4066 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
4068 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
4069 } else if (ins->opcode == OP_LCOMPARE_IMM) {
4070 if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) {
4071 /* The immediate is encoded in two fields */
4072 guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm);
4073 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), "");
4075 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), "");
4078 else if (ins->opcode == OP_COMPARE) {
4079 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
4080 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4082 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
4084 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4086 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
4087 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
4089 * If the target bb contains PHI instructions, LLVM requires
4090 * two PHI entries for this bblock, while we only generate one.
4091 * So convert this to an unconditional bblock. (bxc #171).
4093 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
4095 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
4097 has_terminator = TRUE;
4098 } else if (MONO_IS_SETCC (ins->next)) {
4099 sprintf (dname_buf, "t%d", ins->next->dreg);
4101 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4103 /* Add stores for volatile variables */
4104 emit_volatile_store (ctx, ins->next->dreg);
4105 } else if (MONO_IS_COND_EXC (ins->next)) {
4106 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
4107 CHECK_FAILURE (ctx);
4108 builder = ctx->builder;
4110 LLVM_FAILURE (ctx, "next");
4124 rel = mono_opcode_to_cond (ins->opcode);
4126 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4127 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4138 rel = mono_opcode_to_cond (ins->opcode);
4140 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4141 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4149 gboolean empty = TRUE;
4151 /* Check that all input bblocks really branch to us */
4152 for (i = 0; i < bb->in_count; ++i) {
4153 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4154 ins->inst_phi_args [i + 1] = -1;
4160 /* LLVM doesn't like phi instructions with zero operands */
4161 ctx->is_dead [ins->dreg] = TRUE;
4165 /* Created earlier, insert it now */
4166 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4168 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4169 int sreg1 = ins->inst_phi_args [i + 1];
4173 * Count the number of times the incoming bblock branches to us,
4174 * since llvm requires a separate entry for each.
4176 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4177 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4180 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4181 if (switch_ins->inst_many_bb [j] == bb)
4188 /* Remember for later */
4189 for (j = 0; j < count; ++j) {
4190 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4193 node->in_bb = bb->in_bb [i];
4195 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4205 values [ins->dreg] = lhs;
4209 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
4212 values [ins->dreg] = lhs;
4214 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
4216 * This is added by the spilling pass in case of the JIT,
4217 * but we have to do it ourselves.
4219 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
4223 case OP_MOVE_F_TO_I4: {
4224 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
4227 case OP_MOVE_I4_TO_F: {
4228 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
4231 case OP_MOVE_F_TO_I8: {
4232 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
4235 case OP_MOVE_I8_TO_F: {
4236 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
4269 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4270 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4272 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
4273 CHECK_FAILURE (ctx);
4274 builder = ctx->builder;
4276 switch (ins->opcode) {
4279 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
4283 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
4287 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
4291 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
4295 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
4299 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
4303 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
4307 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4311 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
4315 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
4319 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
4323 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
4327 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
4331 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
4335 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4338 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4341 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4345 g_assert_not_reached ();
4352 lhs = convert (ctx, lhs, LLVMFloatType ());
4353 rhs = convert (ctx, rhs, LLVMFloatType ());
4354 switch (ins->opcode) {
4356 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4359 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4362 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4365 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4368 g_assert_not_reached ();
4377 case OP_IREM_UN_IMM:
4379 case OP_IDIV_UN_IMM:
4385 case OP_ISHR_UN_IMM:
4394 case OP_LSHR_UN_IMM:
4400 case OP_SHR_UN_IMM: {
4403 if (spec [MONO_INST_SRC1] == 'l') {
4404 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4406 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4409 emit_div_check (ctx, builder, bb, ins, lhs, imm);
4410 CHECK_FAILURE (ctx);
4411 builder = ctx->builder;
4413 #if SIZEOF_VOID_P == 4
4414 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
4415 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4418 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4419 lhs = convert (ctx, lhs, IntPtrType ());
4420 imm = convert (ctx, imm, LLVMTypeOf (lhs));
4421 switch (ins->opcode) {
4425 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
4429 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
4433 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
4437 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
4439 case OP_IDIV_UN_IMM:
4440 case OP_LDIV_UN_IMM:
4441 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
4445 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
4447 case OP_IREM_UN_IMM:
4448 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
4453 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
4457 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
4461 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
4466 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
4471 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
4473 case OP_ISHR_UN_IMM:
4474 /* This is used to implement conv.u4, so the lhs could be an i8 */
4475 lhs = convert (ctx, lhs, LLVMInt32Type ());
4476 imm = convert (ctx, imm, LLVMInt32Type ());
4477 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4479 case OP_LSHR_UN_IMM:
4481 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4484 g_assert_not_reached ();
4489 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4492 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
4495 lhs = convert (ctx, lhs, LLVMDoubleType ());
4496 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
4499 lhs = convert (ctx, lhs, LLVMFloatType ());
4500 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
4503 guint32 v = 0xffffffff;
4504 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4508 guint64 v = 0xffffffffffffffffLL;
4509 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
4512 #if defined(TARGET_X86) || defined(TARGET_AMD64)
4514 LLVMValueRef v1, v2;
4516 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
4517 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
4518 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
4523 case OP_ICONV_TO_I1:
4524 case OP_ICONV_TO_I2:
4525 case OP_ICONV_TO_I4:
4526 case OP_ICONV_TO_U1:
4527 case OP_ICONV_TO_U2:
4528 case OP_ICONV_TO_U4:
4529 case OP_LCONV_TO_I1:
4530 case OP_LCONV_TO_I2:
4531 case OP_LCONV_TO_U1:
4532 case OP_LCONV_TO_U2:
4533 case OP_LCONV_TO_U4: {
4536 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
4538 /* Have to do two casts since our vregs have type int */
4539 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
4541 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
4543 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
4546 case OP_ICONV_TO_I8:
4547 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
4549 case OP_ICONV_TO_U8:
4550 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
4552 case OP_FCONV_TO_I4:
4553 case OP_RCONV_TO_I4:
4554 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
4556 case OP_FCONV_TO_I1:
4557 case OP_RCONV_TO_I1:
4558 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
4560 case OP_FCONV_TO_U1:
4561 case OP_RCONV_TO_U1:
4562 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
4564 case OP_FCONV_TO_I2:
4565 case OP_RCONV_TO_I2:
4566 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4568 case OP_FCONV_TO_U2:
4569 case OP_RCONV_TO_U2:
4570 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4572 case OP_FCONV_TO_I8:
4573 case OP_RCONV_TO_I8:
4574 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
4577 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
4579 case OP_ICONV_TO_R8:
4580 case OP_LCONV_TO_R8:
4581 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
4583 case OP_ICONV_TO_R_UN:
4584 case OP_LCONV_TO_R_UN:
4585 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
4587 #if SIZEOF_VOID_P == 4
4590 case OP_LCONV_TO_I4:
4591 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
4593 case OP_ICONV_TO_R4:
4594 case OP_LCONV_TO_R4:
4595 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
4597 values [ins->dreg] = v;
4599 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
4601 case OP_FCONV_TO_R4:
4602 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
4604 values [ins->dreg] = v;
4606 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
4608 case OP_RCONV_TO_R8:
4609 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
4611 case OP_RCONV_TO_R4:
4612 values [ins->dreg] = lhs;
4615 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
4618 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
4621 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
4623 case OP_LOCALLOC_IMM: {
4626 guint32 size = ins->inst_imm;
4627 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
4629 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
4631 if (ins->flags & MONO_INST_INIT) {
4632 LLVMValueRef args [5];
4635 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
4636 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
4637 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
4638 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
4639 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memset_func_name), args, memset_param_count, "");
4642 values [ins->dreg] = v;
4646 LLVMValueRef v, size;
4648 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
4650 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
4652 if (ins->flags & MONO_INST_INIT) {
4653 LLVMValueRef args [5];
4656 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
4658 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
4659 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
4660 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memset_func_name), args, memset_param_count, "");
4662 values [ins->dreg] = v;
4666 case OP_LOADI1_MEMBASE:
4667 case OP_LOADU1_MEMBASE:
4668 case OP_LOADI2_MEMBASE:
4669 case OP_LOADU2_MEMBASE:
4670 case OP_LOADI4_MEMBASE:
4671 case OP_LOADU4_MEMBASE:
4672 case OP_LOADI8_MEMBASE:
4673 case OP_LOADR4_MEMBASE:
4674 case OP_LOADR8_MEMBASE:
4675 case OP_LOAD_MEMBASE:
4683 LLVMValueRef base, index, addr;
4685 gboolean sext = FALSE, zext = FALSE;
4686 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
4688 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
4693 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
4694 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
4699 if (ins->inst_offset == 0) {
4701 } else if (ins->inst_offset % size != 0) {
4702 /* Unaligned load */
4703 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
4704 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
4706 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
4707 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
4711 addr = convert (ctx, addr, LLVMPointerType (t, 0));
4713 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, dname, is_volatile);
4715 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
4717 * These will signal LLVM that these loads do not alias any stores, and
4718 * they can't fail, allowing them to be hoisted out of loops.
4720 set_invariant_load_flag (values [ins->dreg]);
4721 set_metadata_flag (values [ins->dreg], "mono.nofail.load");
4725 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
4727 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
4728 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
4729 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
4733 case OP_STOREI1_MEMBASE_REG:
4734 case OP_STOREI2_MEMBASE_REG:
4735 case OP_STOREI4_MEMBASE_REG:
4736 case OP_STOREI8_MEMBASE_REG:
4737 case OP_STORER4_MEMBASE_REG:
4738 case OP_STORER8_MEMBASE_REG:
4739 case OP_STORE_MEMBASE_REG: {
4741 LLVMValueRef index, addr;
4743 gboolean sext = FALSE, zext = FALSE;
4744 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
4746 if (!values [ins->inst_destbasereg])
4747 LLVM_FAILURE (ctx, "inst_destbasereg");
4749 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
4751 if (ins->inst_offset % size != 0) {
4752 /* Unaligned store */
4753 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
4754 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
4756 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
4757 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
4759 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
4763 case OP_STOREI1_MEMBASE_IMM:
4764 case OP_STOREI2_MEMBASE_IMM:
4765 case OP_STOREI4_MEMBASE_IMM:
4766 case OP_STOREI8_MEMBASE_IMM:
4767 case OP_STORE_MEMBASE_IMM: {
4769 LLVMValueRef index, addr;
4771 gboolean sext = FALSE, zext = FALSE;
4772 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
4774 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
4776 if (ins->inst_offset % size != 0) {
4777 /* Unaligned store */
4778 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
4779 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
4781 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
4782 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
4784 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
4789 emit_load (ctx, bb, &builder, sizeof (gpointer), convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
4791 case OP_OUTARG_VTRETADDR:
4799 case OP_VOIDCALL_MEMBASE:
4800 case OP_CALL_MEMBASE:
4801 case OP_LCALL_MEMBASE:
4802 case OP_FCALL_MEMBASE:
4803 case OP_RCALL_MEMBASE:
4804 case OP_VCALL_MEMBASE:
4805 case OP_VOIDCALL_REG:
4810 case OP_VCALL_REG: {
4811 process_call (ctx, bb, &builder, ins);
4812 CHECK_FAILURE (ctx);
4817 LLVMValueRef indexes [2];
4818 MonoJumpInfo *tmp_ji, *ji;
4819 LLVMValueRef got_entry_addr;
4823 * FIXME: Can't allocate from the cfg mempool since that is freed if
4824 * the LLVM compile fails.
4826 tmp_ji = g_new0 (MonoJumpInfo, 1);
4827 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
4828 tmp_ji->data.target = ins->inst_p0;
4830 ji = mono_aot_patch_info_dup (tmp_ji);
4833 ji->next = cfg->patch_info;
4834 cfg->patch_info = ji;
4836 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
4837 got_offset = mono_aot_get_got_offset (cfg->patch_info);
4838 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
4839 if (!mono_aot_is_shared_got_offset (got_offset)) {
4840 //mono_print_ji (ji);
4842 ctx->has_got_access = TRUE;
4845 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4846 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
4847 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
4849 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
4850 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
4852 set_invariant_load_flag (values [ins->dreg]);
4855 case OP_NOT_REACHED:
4856 LLVMBuildUnreachable (builder);
4857 has_terminator = TRUE;
4858 g_assert (bb->block_num < cfg->max_block_num);
4859 ctx->unreachable [bb->block_num] = TRUE;
4860 /* Might have instructions after this */
4862 MonoInst *next = ins->next;
4864 * FIXME: If later code uses the regs defined by these instructions,
4865 * compilation will fail.
4867 MONO_DELETE_INS (bb, next);
4871 MonoInst *var = ins->inst_i0;
4873 if (var->opcode == OP_VTARG_ADDR) {
4874 /* The variable contains the vtype address */
4875 values [ins->dreg] = values [var->dreg];
4877 values [ins->dreg] = addresses [var->dreg];
4882 LLVMValueRef args [1];
4884 args [0] = convert (ctx, lhs, LLVMDoubleType ());
4885 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "llvm.sin.f64"), args, 1, dname);
4889 LLVMValueRef args [1];
4891 args [0] = convert (ctx, lhs, LLVMDoubleType ());
4892 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "llvm.cos.f64"), args, 1, dname);
4896 LLVMValueRef args [1];
4899 /* This no longer seems to happen */
4901 * LLVM optimizes sqrt(nan) into undefined in
4902 * lib/Analysis/ConstantFolding.cpp
4903 * Also, sqrt(NegativeInfinity) is optimized into 0.
4905 LLVM_FAILURE (ctx, "sqrt");
4907 args [0] = convert (ctx, lhs, LLVMDoubleType ());
4908 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "llvm.sqrt.f64"), args, 1, dname);
4912 LLVMValueRef args [1];
4914 args [0] = convert (ctx, lhs, LLVMDoubleType ());
4915 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "fabs"), args, 1, dname);
4929 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4930 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4932 switch (ins->opcode) {
4935 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
4939 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
4943 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
4947 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
4950 g_assert_not_reached ();
4953 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
4956 case OP_ATOMIC_EXCHANGE_I4:
4957 case OP_ATOMIC_EXCHANGE_I8: {
4958 LLVMValueRef args [2];
4961 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
4962 t = LLVMInt32Type ();
4964 t = LLVMInt64Type ();
4966 g_assert (ins->inst_offset == 0);
4968 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
4969 args [1] = convert (ctx, rhs, t);
4971 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
4974 case OP_ATOMIC_ADD_I4:
4975 case OP_ATOMIC_ADD_I8: {
4976 LLVMValueRef args [2];
4979 if (ins->opcode == OP_ATOMIC_ADD_I4)
4980 t = LLVMInt32Type ();
4982 t = LLVMInt64Type ();
4984 g_assert (ins->inst_offset == 0);
4986 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
4987 args [1] = convert (ctx, rhs, t);
4988 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
4991 case OP_ATOMIC_CAS_I4:
4992 case OP_ATOMIC_CAS_I8: {
4993 LLVMValueRef args [3], val;
4996 if (ins->opcode == OP_ATOMIC_CAS_I4)
4997 t = LLVMInt32Type ();
4999 t = LLVMInt64Type ();
5001 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5003 args [1] = convert (ctx, values [ins->sreg3], t);
5005 args [2] = convert (ctx, values [ins->sreg2], t);
5006 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
5007 /* cmpxchg returns a pair */
5008 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
5011 case OP_MEMORY_BARRIER: {
5012 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
5015 case OP_ATOMIC_LOAD_I1:
5016 case OP_ATOMIC_LOAD_I2:
5017 case OP_ATOMIC_LOAD_I4:
5018 case OP_ATOMIC_LOAD_I8:
5019 case OP_ATOMIC_LOAD_U1:
5020 case OP_ATOMIC_LOAD_U2:
5021 case OP_ATOMIC_LOAD_U4:
5022 case OP_ATOMIC_LOAD_U8:
5023 case OP_ATOMIC_LOAD_R4:
5024 case OP_ATOMIC_LOAD_R8: {
5025 LLVM_FAILURE (ctx, "atomic mono.load intrinsic");
5028 gboolean sext, zext;
5030 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5031 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5032 LLVMValueRef index, addr;
5034 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5039 if (ins->inst_offset != 0) {
5040 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5041 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
5046 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5048 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, dname, is_volatile, barrier);
5051 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5053 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5056 case OP_ATOMIC_STORE_I1:
5057 case OP_ATOMIC_STORE_I2:
5058 case OP_ATOMIC_STORE_I4:
5059 case OP_ATOMIC_STORE_I8:
5060 case OP_ATOMIC_STORE_U1:
5061 case OP_ATOMIC_STORE_U2:
5062 case OP_ATOMIC_STORE_U4:
5063 case OP_ATOMIC_STORE_U8:
5064 case OP_ATOMIC_STORE_R4:
5065 case OP_ATOMIC_STORE_R8: {
5066 LLVM_FAILURE (ctx, "atomic mono.store intrinsic");
5069 gboolean sext, zext;
5071 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5072 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5073 LLVMValueRef index, addr, value;
5075 if (!values [ins->inst_destbasereg])
5076 LLVM_FAILURE (ctx, "inst_destbasereg");
5078 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5080 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5081 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5082 value = convert (ctx, values [ins->sreg1], t);
5084 emit_store_general (ctx, bb, &builder, size, value, addr, is_volatile, barrier);
5087 case OP_RELAXED_NOP: {
5088 #if defined(TARGET_AMD64) || defined(TARGET_X86)
5089 emit_call (ctx, bb, &builder, LLVMGetNamedFunction (ctx->lmodule, "llvm.x86.sse2.pause"), NULL, 0);
5096 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
5098 // 257 == FS segment register
5099 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
5101 // 256 == GS segment register
5102 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5105 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
5106 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
5107 /* See mono_amd64_emit_tls_get () */
5108 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
5110 // 256 == GS segment register
5111 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5112 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
5114 LLVM_FAILURE (ctx, "opcode tls-get");
5119 case OP_TLS_GET_REG: {
5120 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5121 /* See emit_tls_get_reg () */
5122 // 256 == GS segment register
5123 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5124 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt32Type ()), ptrtype, ""), "");
5126 LLVM_FAILURE (ctx, "opcode tls-get");
5131 case OP_TLS_SET_REG: {
5132 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5133 /* See emit_tls_get_reg () */
5134 // 256 == GS segment register
5135 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5136 LLVMBuildStore (builder, convert (ctx, lhs, IntPtrType ()), LLVMBuildIntToPtr (builder, convert (ctx, rhs, LLVMInt32Type ()), ptrtype, ""));
5138 LLVM_FAILURE (ctx, "opcode tls-set-reg");
5147 case OP_IADD_OVF_UN:
5149 case OP_ISUB_OVF_UN:
5151 case OP_IMUL_OVF_UN:
5152 #if SIZEOF_VOID_P == 8
5154 case OP_LADD_OVF_UN:
5156 case OP_LSUB_OVF_UN:
5158 case OP_LMUL_OVF_UN:
5161 LLVMValueRef args [2], val, ovf, func;
5163 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
5164 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
5165 func = LLVMGetNamedFunction (lmodule, ovf_op_to_intrins (ins->opcode));
5167 val = LLVMBuildCall (builder, func, args, 2, "");
5168 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
5169 ovf = LLVMBuildExtractValue (builder, val, 1, "");
5170 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
5171 CHECK_FAILURE (ctx);
5172 builder = ctx->builder;
5178 * We currently model them using arrays. Promotion to local vregs is
5179 * disabled for them in mono_handle_global_vregs () in the LLVM case,
5180 * so we always have an entry in cfg->varinfo for them.
5181 * FIXME: Is this needed ?
5184 MonoClass *klass = ins->klass;
5185 LLVMValueRef args [5];
5189 LLVM_FAILURE (ctx, "!klass");
5193 if (!addresses [ins->dreg])
5194 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5195 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5196 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5197 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5199 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5200 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5201 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memset_func_name), args, memset_param_count, "");
5204 case OP_DUMMY_VZERO:
5207 case OP_STOREV_MEMBASE:
5208 case OP_LOADV_MEMBASE:
5210 MonoClass *klass = ins->klass;
5211 LLVMValueRef src = NULL, dst, args [5];
5212 gboolean done = FALSE;
5216 LLVM_FAILURE (ctx, "!klass");
5220 if (mini_is_gsharedvt_klass (klass)) {
5222 LLVM_FAILURE (ctx, "gsharedvt");
5226 switch (ins->opcode) {
5227 case OP_STOREV_MEMBASE:
5228 if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg &&
5229 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
5230 /* Decomposed earlier */
5231 g_assert_not_reached ();
5234 if (!addresses [ins->sreg1]) {
5236 g_assert (values [ins->sreg1]);
5237 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
5238 LLVMBuildStore (builder, values [ins->sreg1], dst);
5241 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5242 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5245 case OP_LOADV_MEMBASE:
5246 if (!addresses [ins->dreg])
5247 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5248 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5249 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5252 if (!addresses [ins->sreg1])
5253 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
5254 if (!addresses [ins->dreg])
5255 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5256 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5257 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5260 g_assert_not_reached ();
5262 CHECK_FAILURE (ctx);
5269 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5270 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5272 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5273 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5274 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memcpy_func_name), args, memcpy_param_count, "");
5277 case OP_LLVM_OUTARG_VT: {
5278 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
5279 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
5281 if (ainfo->storage == LLVMArgScalarByRef) {
5282 LLVMTypeRef argtype;
5283 LLVMValueRef loc, v;
5285 argtype = type_to_llvm_arg_type (ctx, t);
5286 loc = build_alloca_llvm_type (ctx, argtype, 0);
5287 v = convert (ctx, values [ins->sreg1], argtype);
5288 LLVMBuildStore (ctx->builder, v, loc);
5289 addresses [ins->dreg] = loc;
5291 if (!addresses [ins->sreg1]) {
5292 addresses [ins->sreg1] = build_alloca (ctx, t);
5293 g_assert (values [ins->sreg1]);
5294 LLVMBuildStore (builder, values [ins->sreg1], addresses [ins->sreg1]);
5296 addresses [ins->dreg] = addresses [ins->sreg1];
5304 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5306 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
5309 case OP_LOADX_MEMBASE: {
5310 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
5313 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5314 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
5317 case OP_STOREX_MEMBASE: {
5318 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
5321 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5322 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
5329 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
5333 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
5339 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
5343 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
5347 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
5351 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
5354 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
5357 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
5360 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
5364 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
5375 LLVMValueRef v = NULL;
5377 switch (ins->opcode) {
5382 t = LLVMVectorType (LLVMInt32Type (), 4);
5383 rt = LLVMVectorType (LLVMFloatType (), 4);
5389 t = LLVMVectorType (LLVMInt64Type (), 2);
5390 rt = LLVMVectorType (LLVMDoubleType (), 2);
5393 t = LLVMInt32Type ();
5394 rt = LLVMInt32Type ();
5395 g_assert_not_reached ();
5398 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5399 rhs = LLVMBuildBitCast (builder, rhs, t, "");
5400 switch (ins->opcode) {
5403 v = LLVMBuildAnd (builder, lhs, rhs, "");
5407 v = LLVMBuildOr (builder, lhs, rhs, "");
5411 v = LLVMBuildXor (builder, lhs, rhs, "");
5415 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
5418 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
5442 case OP_PADDB_SAT_UN:
5443 case OP_PADDW_SAT_UN:
5444 case OP_PSUBB_SAT_UN:
5445 case OP_PSUBW_SAT_UN:
5453 case OP_PMULW_HIGH_UN: {
5454 LLVMValueRef args [2];
5459 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5466 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, ""), LLVMTypeOf (lhs), "");
5470 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, ""), LLVMTypeOf (lhs), "");
5478 case OP_EXTRACTX_U2:
5480 case OP_EXTRACT_U1: {
5482 gboolean zext = FALSE;
5484 t = simd_op_to_llvm_type (ins->opcode);
5486 switch (ins->opcode) {
5494 case OP_EXTRACTX_U2:
5499 t = LLVMInt32Type ();
5500 g_assert_not_reached ();
5503 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5504 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
5506 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
5515 case OP_EXPAND_R8: {
5516 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
5517 LLVMValueRef mask [16], v;
5520 for (i = 0; i < 16; ++i)
5521 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5523 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
5525 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5526 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
5531 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5534 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5537 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5540 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5543 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5546 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5557 case OP_EXTRACT_MASK:
5564 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
5566 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
5572 LLVMValueRef args [3];
5576 args [2] = LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE);
5578 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 3, dname);
5583 /* This is only used for implementing shifts by non-immediate */
5584 values [ins->dreg] = lhs;
5595 LLVMValueRef args [3];
5598 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5600 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5611 case OP_PSHLQ_REG: {
5612 LLVMValueRef args [3];
5615 args [1] = values [ins->sreg2];
5617 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5624 case OP_PSHUFLEW_LOW:
5625 case OP_PSHUFLEW_HIGH: {
5627 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
5628 int i, mask_size = 0;
5629 int imask = ins->inst_c0;
5631 /* Convert the x86 shuffle mask to LLVM's */
5632 switch (ins->opcode) {
5635 mask [0] = ((imask >> 0) & 3);
5636 mask [1] = ((imask >> 2) & 3);
5637 mask [2] = ((imask >> 4) & 3) + 4;
5638 mask [3] = ((imask >> 6) & 3) + 4;
5639 v1 = values [ins->sreg1];
5640 v2 = values [ins->sreg2];
5644 mask [0] = ((imask >> 0) & 1);
5645 mask [1] = ((imask >> 1) & 1) + 2;
5646 v1 = values [ins->sreg1];
5647 v2 = values [ins->sreg2];
5649 case OP_PSHUFLEW_LOW:
5651 mask [0] = ((imask >> 0) & 3);
5652 mask [1] = ((imask >> 2) & 3);
5653 mask [2] = ((imask >> 4) & 3);
5654 mask [3] = ((imask >> 6) & 3);
5659 v1 = values [ins->sreg1];
5660 v2 = LLVMGetUndef (LLVMTypeOf (v1));
5662 case OP_PSHUFLEW_HIGH:
5668 mask [4] = 4 + ((imask >> 0) & 3);
5669 mask [5] = 4 + ((imask >> 2) & 3);
5670 mask [6] = 4 + ((imask >> 4) & 3);
5671 mask [7] = 4 + ((imask >> 6) & 3);
5672 v1 = values [ins->sreg1];
5673 v2 = LLVMGetUndef (LLVMTypeOf (v1));
5677 mask [0] = ((imask >> 0) & 3);
5678 mask [1] = ((imask >> 2) & 3);
5679 mask [2] = ((imask >> 4) & 3);
5680 mask [3] = ((imask >> 6) & 3);
5681 v1 = values [ins->sreg1];
5682 v2 = LLVMGetUndef (LLVMTypeOf (v1));
5685 g_assert_not_reached ();
5687 for (i = 0; i < mask_size; ++i)
5688 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
5690 values [ins->dreg] =
5691 LLVMBuildShuffleVector (builder, v1, v2,
5692 LLVMConstVector (mask_values, mask_size), dname);
5696 case OP_UNPACK_LOWB:
5697 case OP_UNPACK_LOWW:
5698 case OP_UNPACK_LOWD:
5699 case OP_UNPACK_LOWQ:
5700 case OP_UNPACK_LOWPS:
5701 case OP_UNPACK_LOWPD:
5702 case OP_UNPACK_HIGHB:
5703 case OP_UNPACK_HIGHW:
5704 case OP_UNPACK_HIGHD:
5705 case OP_UNPACK_HIGHQ:
5706 case OP_UNPACK_HIGHPS:
5707 case OP_UNPACK_HIGHPD: {
5709 LLVMValueRef mask_values [16];
5710 int i, mask_size = 0;
5711 gboolean low = FALSE;
5713 switch (ins->opcode) {
5714 case OP_UNPACK_LOWB:
5718 case OP_UNPACK_LOWW:
5722 case OP_UNPACK_LOWD:
5723 case OP_UNPACK_LOWPS:
5727 case OP_UNPACK_LOWQ:
5728 case OP_UNPACK_LOWPD:
5732 case OP_UNPACK_HIGHB:
5735 case OP_UNPACK_HIGHW:
5738 case OP_UNPACK_HIGHD:
5739 case OP_UNPACK_HIGHPS:
5742 case OP_UNPACK_HIGHQ:
5743 case OP_UNPACK_HIGHPD:
5747 g_assert_not_reached ();
5751 for (i = 0; i < (mask_size / 2); ++i) {
5753 mask [(i * 2) + 1] = mask_size + i;
5756 for (i = 0; i < (mask_size / 2); ++i) {
5757 mask [(i * 2)] = (mask_size / 2) + i;
5758 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
5762 for (i = 0; i < mask_size; ++i)
5763 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
5765 values [ins->dreg] =
5766 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
5767 LLVMConstVector (mask_values, mask_size), dname);
5772 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
5773 LLVMValueRef v, val;
5775 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5776 val = LLVMConstNull (t);
5777 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5778 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
5780 values [ins->dreg] = val;
5784 case OP_DUPPS_HIGH: {
5785 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
5786 LLVMValueRef v1, v2, val;
5789 if (ins->opcode == OP_DUPPS_LOW) {
5790 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5791 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
5793 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
5794 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
5796 val = LLVMConstNull (t);
5797 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5798 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
5799 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
5800 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
5802 values [ins->dreg] = val;
5812 * EXCEPTION HANDLING
5814 case OP_IMPLICIT_EXCEPTION:
5815 /* This marks a place where an implicit exception can happen */
5816 if (bb->region != -1)
5817 LLVM_FAILURE (ctx, "implicit-exception");
5821 gboolean rethrow = (ins->opcode == OP_RETHROW);
5822 if (ctx->llvm_only) {
5823 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
5824 has_terminator = TRUE;
5825 ctx->unreachable [bb->block_num] = TRUE;
5827 emit_throw (ctx, bb, rethrow, lhs);
5828 builder = ctx->builder;
5832 case OP_CALL_HANDLER: {
5834 * We don't 'call' handlers, but instead simply branch to them.
5835 * The code generated by ENDFINALLY will branch back to us.
5837 LLVMBasicBlockRef noex_bb;
5839 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
5841 bb_list = info->call_handler_return_bbs;
5844 * Set the indicator variable for the finally clause.
5846 lhs = info->finally_ind;
5848 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
5850 /* Branch to the finally clause */
5851 LLVMBuildBr (builder, info->call_handler_target_bb);
5853 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
5854 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
5856 builder = ctx->builder = create_builder (ctx);
5857 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
5859 bblocks [bb->block_num].end_bblock = noex_bb;
5862 case OP_START_HANDLER: {
5865 case OP_ENDFINALLY: {
5866 LLVMBasicBlockRef resume_bb;
5867 MonoBasicBlock *handler_bb;
5868 LLVMValueRef val, switch_ins, callee;
5872 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
5873 g_assert (handler_bb);
5874 info = &bblocks [handler_bb->block_num];
5875 lhs = info->finally_ind;
5878 bb_list = info->call_handler_return_bbs;
5880 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
5882 /* Load the finally variable */
5883 val = LLVMBuildLoad (builder, lhs, "");
5885 /* Reset the variable */
5886 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
5888 /* Branch to either resume_bb, or to the bblocks in bb_list */
5889 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
5891 * The other targets are added at the end to handle OP_CALL_HANDLER
5892 * opcodes processed later.
5894 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
5896 builder = ctx->builder = create_builder (ctx);
5897 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
5899 if (ctx->llvm_only) {
5900 emit_resume_eh (ctx, bb);
5902 if (ctx->cfg->compile_aot) {
5903 callee = get_callee (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline");
5905 callee = LLVMGetNamedFunction (lmodule, "llvm_resume_unwind_trampoline");
5907 LLVMBuildCall (builder, callee, NULL, 0, "");
5908 LLVMBuildUnreachable (builder);
5911 has_terminator = TRUE;
5917 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
5918 LLVM_FAILURE (ctx, reason);
5923 /* Convert the value to the type required by phi nodes */
5924 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
5925 if (!values [ins->dreg])
5927 values [ins->dreg] = addresses [ins->dreg];
5929 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
5932 /* Add stores for volatile variables */
5933 if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
5934 emit_volatile_store (ctx, ins->dreg);
5937 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
5938 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
5941 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
5942 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
5943 LLVMBuildRetVoid (builder);
5946 if (bb == cfg->bb_entry)
5947 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
5956 * mono_llvm_check_method_supported:
5958 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
5959 * compiling a method twice.
5962 mono_llvm_check_method_supported (MonoCompile *cfg)
5969 if (cfg->method->save_lmf) {
5970 cfg->exception_message = g_strdup ("lmf");
5971 cfg->disable_llvm = TRUE;
5973 if (cfg->disable_llvm)
5977 * Nested clauses where one of the clauses is a finally clause is
5978 * not supported, because LLVM can't figure out the control flow,
5979 * probably because we resume exception handling by calling our
5980 * own function instead of using the 'resume' llvm instruction.
5982 for (i = 0; i < cfg->header->num_clauses; ++i) {
5983 for (j = 0; j < cfg->header->num_clauses; ++j) {
5984 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
5985 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
5987 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset &&
5988 (clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
5989 cfg->exception_message = g_strdup ("nested clauses");
5990 cfg->disable_llvm = TRUE;
5995 if (cfg->disable_llvm)
5999 if (cfg->method->dynamic) {
6000 cfg->exception_message = g_strdup ("dynamic.");
6001 cfg->disable_llvm = TRUE;
6003 if (cfg->disable_llvm)
6007 static LLVMCallInfo*
6008 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
6010 LLVMCallInfo *linfo;
6013 linfo = mono_arch_get_llvm_call_info (cfg, sig);
6014 for (i = 0; i < sig->param_count; ++i)
6015 linfo->args [i + sig->hasthis].type = sig->params [i];
6021 * mono_llvm_emit_method:
6023 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
6026 mono_llvm_emit_method (MonoCompile *cfg)
6029 MonoMethodSignature *sig;
6031 LLVMTypeRef method_type;
6032 LLVMValueRef method = NULL;
6034 LLVMValueRef *values;
6035 int i, max_block_num, bb_index;
6036 gboolean last = FALSE;
6037 GPtrArray *phi_values;
6038 LLVMCallInfo *linfo;
6040 LLVMModuleRef lmodule;
6042 GPtrArray *bblock_list;
6043 MonoMethodHeader *header;
6044 MonoExceptionClause *clause;
6047 /* The code below might acquire the loader lock, so use it for global locking */
6048 mono_loader_lock ();
6050 /* Used to communicate with the callbacks */
6051 mono_native_tls_set_value (current_cfg_tls_id, cfg);
6053 ctx = g_new0 (EmitContext, 1);
6055 ctx->mempool = cfg->mempool;
6058 * This maps vregs to the LLVM instruction defining them
6060 values = g_new0 (LLVMValueRef, cfg->next_vreg);
6062 * This maps vregs for volatile variables to the LLVM instruction defining their
6065 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
6066 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
6067 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
6068 phi_values = g_ptr_array_sized_new (256);
6070 * This signals whenever the vreg was defined by a phi node with no input vars
6071 * (i.e. all its input bblocks end with NOT_REACHABLE).
6073 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
6074 /* Whenever the bblock is unreachable */
6075 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
6076 bblock_list = g_ptr_array_sized_new (256);
6078 ctx->values = values;
6079 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
6080 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
6081 ctx->method_to_callers = g_hash_table_new (NULL, NULL);
6083 if (cfg->compile_aot) {
6084 ctx->module = &aot_module;
6085 method_name = mono_aot_get_method_name (cfg);
6086 cfg->llvm_method_name = g_strdup (method_name);
6088 init_jit_module (cfg->domain);
6089 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
6090 method_name = mono_method_full_name (cfg->method, TRUE);
6093 lmodule = ctx->lmodule = ctx->module->lmodule;
6094 ctx->llvm_only = ctx->module->llvm_only;
6097 LLVM_FAILURE (ctx, "gsharedvt");
6101 static int count = 0;
6104 if (g_getenv ("LLVM_COUNT")) {
6105 if (count == atoi (g_getenv ("LLVM_COUNT"))) {
6106 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
6110 if (count > atoi (g_getenv ("LLVM_COUNT")))
6111 LLVM_FAILURE (ctx, "");
6116 sig = mono_method_signature (cfg->method);
6119 linfo = get_llvm_call_info (cfg, sig);
6121 CHECK_FAILURE (ctx);
6124 linfo->rgctx_arg = TRUE;
6125 method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
6126 CHECK_FAILURE (ctx);
6128 method = LLVMAddFunction (lmodule, method_name, method_type);
6129 ctx->lmethod = method;
6131 if (!cfg->llvm_only)
6132 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
6133 LLVMSetLinkage (method, LLVMPrivateLinkage);
6135 LLVMAddFunctionAttr (method, LLVMUWTable);
6137 if (cfg->compile_aot) {
6138 LLVMSetLinkage (method, LLVMInternalLinkage);
6139 if (ctx->module->external_symbols) {
6140 LLVMSetLinkage (method, LLVMExternalLinkage);
6141 LLVMSetVisibility (method, LLVMHiddenVisibility);
6144 LLVMSetLinkage (method, LLVMPrivateLinkage);
6147 if (cfg->method->save_lmf && !cfg->llvm_only)
6148 LLVM_FAILURE (ctx, "lmf");
6150 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only)
6151 LLVM_FAILURE (ctx, "pinvoke signature");
6153 header = cfg->header;
6154 for (i = 0; i < header->num_clauses; ++i) {
6155 clause = &header->clauses [i];
6156 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE)
6157 LLVM_FAILURE (ctx, "non-finally/catch clause.");
6159 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING))
6160 /* We can't handle inlined methods with clauses */
6161 LLVMAddFunctionAttr (method, LLVMNoInlineAttribute);
6163 if (linfo->rgctx_arg) {
6164 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
6165 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
6167 * We mark the rgctx parameter with the inreg attribute, which is mapped to
6168 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
6169 * CC_X86_64_Mono in X86CallingConv.td.
6171 if (!ctx->llvm_only)
6172 LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute);
6173 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
6175 ctx->rgctx_arg_pindex = -1;
6177 if (cfg->vret_addr) {
6178 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
6179 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
6180 if (linfo->ret.storage == LLVMArgVtypeByRef) {
6181 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMStructRetAttribute);
6182 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
6184 } else if (linfo->ret.storage == LLVMArgScalarRetAddr) {
6185 LLVMValueRef param = LLVMGetParam (method, linfo->vret_arg_pindex);
6186 LLVMSetValueName (param, "vret");
6190 ctx->this_arg_pindex = linfo->this_arg_pindex;
6191 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
6192 values [cfg->args [0]->dreg] = ctx->this_arg;
6193 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
6196 names = g_new (char *, sig->param_count);
6197 mono_method_get_param_names (cfg->method, (const char **) names);
6199 for (i = 0; i < sig->param_count; ++i) {
6200 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
6203 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, ainfo->pindex);
6204 if (ainfo->storage == LLVMArgScalarByRef) {
6205 if (names [i] && names [i][0] != '\0')
6206 name = g_strdup_printf ("p_arg_%s", names [i]);
6208 name = g_strdup_printf ("p_arg_%d", i);
6210 if (names [i] && names [i][0] != '\0')
6211 name = g_strdup_printf ("arg_%s", names [i]);
6213 name = g_strdup_printf ("arg_%d", i);
6215 LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name);
6217 if (ainfo->storage == LLVMArgVtypeByVal)
6218 LLVMAddAttribute (LLVMGetParam (method, ainfo->pindex), LLVMByValAttribute);
6220 if (ainfo->storage == LLVMArgVtypeByRef) {
6222 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
6227 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
6228 ctx->minfo = mono_debug_lookup_method (cfg->method);
6229 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, method_name);
6233 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
6234 max_block_num = MAX (max_block_num, bb->block_num);
6235 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
6237 /* Add branches between non-consecutive bblocks */
6238 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6239 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
6240 bb->next_bb != bb->last_ins->inst_false_bb) {
6242 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
6243 inst->opcode = OP_BR;
6244 inst->inst_target_bb = bb->last_ins->inst_false_bb;
6245 mono_bblock_add_inst (bb, inst);
6250 * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR
6251 * was later optimized away, so clear these flags, and add them back for the still
6252 * present OP_LDADDR instructions.
6254 for (i = 0; i < cfg->next_vreg; ++i) {
6257 ins = get_vreg_to_inst (cfg, i);
6258 if (ins && ins != cfg->rgctx_var)
6259 ins->flags &= ~MONO_INST_INDIRECT;
6263 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
6265 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6267 LLVMBuilderRef builder;
6269 char dname_buf[128];
6271 builder = create_builder (ctx);
6273 for (ins = bb->code; ins; ins = ins->next) {
6274 switch (ins->opcode) {
6279 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, &ins->klass->byval_arg));
6281 CHECK_FAILURE (ctx);
6283 if (ins->opcode == OP_VPHI) {
6284 /* Treat valuetype PHI nodes as operating on the address itself */
6285 g_assert (ins->klass);
6286 phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
6290 * Have to precreate these, as they can be referenced by
6291 * earlier instructions.
6293 sprintf (dname_buf, "t%d", ins->dreg);
6295 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
6297 if (ins->opcode == OP_VPHI)
6298 ctx->addresses [ins->dreg] = values [ins->dreg];
6300 g_ptr_array_add (phi_values, values [ins->dreg]);
6303 * Set the expected type of the incoming arguments since these have
6304 * to have the same type.
6306 for (i = 0; i < ins->inst_phi_args [0]; i++) {
6307 int sreg1 = ins->inst_phi_args [i + 1];
6310 ctx->vreg_types [sreg1] = phi_type;
6315 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
6324 * Create an ordering for bblocks, use the depth first order first, then
6325 * put the exception handling bblocks last.
6327 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
6328 bb = cfg->bblocks [bb_index];
6329 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
6330 g_ptr_array_add (bblock_list, bb);
6331 bblocks [bb->block_num].added = TRUE;
6335 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6336 if (!bblocks [bb->block_num].added)
6337 g_ptr_array_add (bblock_list, bb);
6341 * Second pass: generate code.
6344 LLVMBuilderRef entry_builder = create_builder (ctx);
6345 LLVMBasicBlockRef entry_bb = get_bb (ctx, cfg->bb_entry);
6346 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
6347 emit_entry_bb (ctx, entry_builder);
6349 // Make landing pads first
6350 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
6352 if (ctx->llvm_only) {
6353 size_t group_index = 0;
6354 while (group_index < cfg->header->num_clauses) {
6356 size_t cursor = group_index;
6357 while (cursor < cfg->header->num_clauses &&
6358 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
6359 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
6364 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
6365 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
6366 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
6368 group_index = cursor;
6372 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
6373 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
6375 // Prune unreachable mono BBs.
6376 if (!(bb == cfg->bb_entry || bb->in_count > 0))
6379 process_bb (ctx, bb);
6380 CHECK_FAILURE (ctx);
6382 g_hash_table_destroy (ctx->exc_meta);
6384 mono_memory_barrier ();
6386 /* Add incoming phi values */
6387 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6388 GSList *l, *ins_list;
6390 ins_list = bblocks [bb->block_num].phi_nodes;
6392 for (l = ins_list; l; l = l->next) {
6393 PhiNode *node = (PhiNode*)l->data;
6394 MonoInst *phi = node->phi;
6395 int sreg1 = node->sreg;
6396 LLVMBasicBlockRef in_bb;
6401 in_bb = get_end_bb (ctx, node->in_bb);
6403 if (ctx->unreachable [node->in_bb->block_num])
6406 if (!values [sreg1])
6407 /* Can happen with values in EH clauses */
6408 LLVM_FAILURE (ctx, "incoming phi sreg1");
6410 if (phi->opcode == OP_VPHI) {
6411 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
6412 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
6414 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg]))
6416 LLVM_FAILURE (ctx, "incoming phi arg type mismatch");
6417 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
6418 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
6423 /* Nullify empty phi instructions */
6424 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6425 GSList *l, *ins_list;
6427 ins_list = bblocks [bb->block_num].phi_nodes;
6429 for (l = ins_list; l; l = l->next) {
6430 PhiNode *node = (PhiNode*)l->data;
6431 MonoInst *phi = node->phi;
6432 LLVMValueRef phi_ins = values [phi->dreg];
6435 /* Already removed */
6438 if (LLVMCountIncoming (phi_ins) == 0) {
6439 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
6440 LLVMInstructionEraseFromParent (phi_ins);
6441 values [phi->dreg] = NULL;
6446 /* Create the SWITCH statements for ENDFINALLY instructions */
6447 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6448 BBInfo *info = &bblocks [bb->block_num];
6450 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
6451 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
6452 GSList *bb_list = info->call_handler_return_bbs;
6454 for (i = 0; i < g_slist_length (bb_list); ++i)
6455 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data);
6459 /* Initialize the method if needed */
6460 if (cfg->compile_aot && ctx->llvm_only) {
6461 // FIXME: Add more shared got entries
6462 ctx->builder = create_builder (ctx);
6463 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
6465 // FIXME: beforefieldinit
6466 if (ctx->has_got_access || mono_class_get_cctor (cfg->method->klass)) {
6467 emit_init_method (ctx);
6469 LLVMBuildBr (ctx->builder, ctx->inited_bb);
6473 if (cfg->llvm_only) {
6474 GHashTableIter iter;
6476 GSList *callers, *l, *l2;
6479 * Add the contents of ctx->method_to_callers to module->method_to_callers.
6480 * We can't do this earlier, as it contains llvm instructions which can be
6481 * freed if compilation fails.
6482 * FIXME: Get rid of this when all methods can be llvm compiled.
6484 g_hash_table_iter_init (&iter, ctx->method_to_callers);
6485 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
6486 for (l = callers; l; l = l->next) {
6487 l2 = g_hash_table_lookup (ctx->module->method_to_callers, method);
6488 l2 = g_slist_prepend (l2, l->data);
6489 g_hash_table_insert (ctx->module->method_to_callers, method, l2);
6494 if (cfg->verbose_level > 1)
6495 mono_llvm_dump_value (method);
6497 if (cfg->compile_aot && !cfg->llvm_only)
6498 mark_as_used (ctx->module, method);
6500 if (cfg->compile_aot) {
6501 LLVMValueRef md_args [16];
6502 LLVMValueRef md_node;
6505 method_index = mono_aot_get_method_index (cfg->orig_method);
6506 md_args [0] = LLVMMDString (method_name, strlen (method_name));
6507 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
6508 md_node = LLVMMDNode (md_args, 2);
6509 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
6510 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
6513 if (cfg->compile_aot) {
6514 /* Don't generate native code, keep the LLVM IR */
6515 if (cfg->verbose_level)
6516 printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), method_name);
6518 int err = LLVMVerifyFunction(method, LLVMPrintMessageAction);
6519 g_assert (err == 0);
6521 LLVMVerifyFunction(method, 0);
6522 mono_llvm_optimize_method (ctx->module->mono_ee, method);
6524 if (cfg->verbose_level > 1)
6525 mono_llvm_dump_value (method);
6527 cfg->native_code = LLVMGetPointerToGlobal (ctx->module->ee, method);
6529 /* Set by emit_cb */
6530 g_assert (cfg->code_len);
6532 /* FIXME: Free the LLVM IL for the function */
6535 if (ctx->module->method_to_lmethod)
6536 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, method);
6537 if (ctx->module->idx_to_lmethod)
6538 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), method);
6540 if (ctx->llvm_only && cfg->orig_method->klass->valuetype && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
6541 emit_unbox_tramp (ctx, method_name, method_type, method, cfg->method_index);
6548 /* Need to add unused phi nodes as they can be referenced by other values */
6549 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (method, "PHI_BB");
6550 LLVMBuilderRef builder;
6552 builder = create_builder (ctx);
6553 LLVMPositionBuilderAtEnd (builder, phi_bb);
6555 for (i = 0; i < phi_values->len; ++i) {
6556 LLVMValueRef v = g_ptr_array_index (phi_values, i);
6557 if (LLVMGetInstructionParent (v) == NULL)
6558 LLVMInsertIntoBuilder (builder, v);
6561 LLVMDeleteFunction (method);
6566 g_free (ctx->addresses);
6567 g_free (ctx->vreg_types);
6568 g_free (ctx->vreg_cli_types);
6569 g_free (ctx->is_dead);
6570 g_free (ctx->unreachable);
6571 g_ptr_array_free (phi_values, TRUE);
6572 g_free (ctx->bblocks);
6573 g_hash_table_destroy (ctx->region_to_handler);
6574 g_hash_table_destroy (ctx->clause_to_handler);
6575 g_free (method_name);
6576 g_ptr_array_free (bblock_list, TRUE);
6578 for (l = ctx->builders; l; l = l->next) {
6579 LLVMBuilderRef builder = l->data;
6580 LLVMDisposeBuilder (builder);
6585 mono_native_tls_set_value (current_cfg_tls_id, NULL);
6587 mono_loader_unlock ();
6591 * mono_llvm_emit_call:
6593 * Same as mono_arch_emit_call () for LLVM.
6596 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
6599 MonoMethodSignature *sig;
6600 int i, n, stack_size;
6605 sig = call->signature;
6606 n = sig->param_count + sig->hasthis;
6608 call->cinfo = get_llvm_call_info (cfg, sig);
6610 if (cfg->disable_llvm)
6613 if (sig->call_convention == MONO_CALL_VARARG) {
6614 cfg->exception_message = g_strdup ("varargs");
6615 cfg->disable_llvm = TRUE;
6618 for (i = 0; i < n; ++i) {
6621 ainfo = call->cinfo->args + i;
6623 in = call->args [i];
6625 /* Simply remember the arguments */
6626 switch (ainfo->storage) {
6627 case LLVMArgNormal: {
6628 MonoType *t = (sig->hasthis && i == 0) ? &mono_get_intptr_class ()->byval_arg : ainfo->type;
6631 opcode = mono_type_to_regmove (cfg, t);
6632 if (opcode == OP_FMOVE) {
6633 MONO_INST_NEW (cfg, ins, OP_FMOVE);
6634 ins->dreg = mono_alloc_freg (cfg);
6635 } else if (opcode == OP_LMOVE) {
6636 MONO_INST_NEW (cfg, ins, OP_LMOVE);
6637 ins->dreg = mono_alloc_lreg (cfg);
6638 } else if (opcode == OP_RMOVE) {
6639 MONO_INST_NEW (cfg, ins, OP_RMOVE);
6640 ins->dreg = mono_alloc_freg (cfg);
6642 MONO_INST_NEW (cfg, ins, OP_MOVE);
6643 ins->dreg = mono_alloc_ireg (cfg);
6645 ins->sreg1 = in->dreg;
6648 case LLVMArgVtypeByVal:
6649 case LLVMArgVtypeByRef:
6650 case LLVMArgVtypeInReg:
6651 case LLVMArgVtypeAsScalar:
6652 case LLVMArgScalarByRef:
6653 case LLVMArgAsIArgs:
6654 case LLVMArgAsFpArgs:
6655 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
6656 ins->dreg = mono_alloc_ireg (cfg);
6657 ins->sreg1 = in->dreg;
6658 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
6659 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
6660 ins->inst_vtype = ainfo->type;
6661 ins->klass = mono_class_from_mono_type (ainfo->type);
6664 cfg->exception_message = g_strdup ("ainfo->storage");
6665 cfg->disable_llvm = TRUE;
6669 if (!cfg->disable_llvm) {
6670 MONO_ADD_INS (cfg->cbb, ins);
6671 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
6676 static unsigned char*
6677 alloc_cb (LLVMValueRef function, int size)
6681 cfg = mono_native_tls_get_value (current_cfg_tls_id);
6685 return mono_domain_code_reserve (cfg->domain, size);
6687 return mono_domain_code_reserve (mono_domain_get (), size);
6692 emitted_cb (LLVMValueRef function, void *start, void *end)
6696 cfg = mono_native_tls_get_value (current_cfg_tls_id);
6698 cfg->code_len = (guint8*)end - (guint8*)start;
6702 exception_cb (void *data)
6705 MonoJitExceptionInfo *ei;
6706 guint32 ei_len, i, j, nested_len, nindex;
6707 gpointer *type_info;
6708 int this_reg, this_offset;
6710 cfg = mono_native_tls_get_value (current_cfg_tls_id);
6714 * data points to a DWARF FDE structure, convert it to our unwind format and
6716 * An alternative would be to save it directly, and modify our unwinder to work
6719 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
6720 if (cfg->verbose_level > 1)
6721 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
6723 /* Count nested clauses */
6725 for (i = 0; i < ei_len; ++i) {
6726 gint32 cindex1 = *(gint32*)type_info [i];
6727 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
6729 for (j = 0; j < cfg->header->num_clauses; ++j) {
6731 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
6733 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
6739 cfg->llvm_ex_info = mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
6740 cfg->llvm_ex_info_len = ei_len + nested_len;
6741 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
6742 /* Fill the rest of the information from the type info */
6743 for (i = 0; i < ei_len; ++i) {
6744 gint32 clause_index = *(gint32*)type_info [i];
6745 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
6747 cfg->llvm_ex_info [i].flags = clause->flags;
6748 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
6749 cfg->llvm_ex_info [i].clause_index = clause_index;
6753 * For nested clauses, the LLVM produced exception info associates the try interval with
6754 * the innermost handler, while mono expects it to be associated with all nesting clauses.
6755 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
6756 * and everything else from the nested clause.
6759 for (i = 0; i < ei_len; ++i) {
6760 gint32 cindex1 = *(gint32*)type_info [i];
6761 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
6763 for (j = 0; j < cfg->header->num_clauses; ++j) {
6765 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
6766 MonoJitExceptionInfo *nesting_ei, *nested_ei;
6768 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
6769 /* clause1 is the nested clause */
6770 nested_ei = &cfg->llvm_ex_info [i];
6771 nesting_ei = &cfg->llvm_ex_info [nindex];
6774 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
6776 nesting_ei->flags = clause2->flags;
6777 nesting_ei->data.catch_class = clause2->data.catch_class;
6778 nesting_ei->clause_index = cindex2;
6782 g_assert (nindex == ei_len + nested_len);
6783 cfg->llvm_this_reg = this_reg;
6784 cfg->llvm_this_offset = this_offset;
6786 /* type_info [i] is cfg mempool allocated, no need to free it */
6793 dlsym_cb (const char *name, void **symbol)
6799 if (!strcmp (name, "__bzero")) {
6800 *symbol = (void*)bzero;
6802 current = mono_dl_open (NULL, 0, NULL);
6805 err = mono_dl_symbol (current, name, symbol);
6807 mono_dl_close (current);
6809 #ifdef MONO_ARCH_HAVE_CREATE_LLVM_NATIVE_THUNK
6810 *symbol = (char*)mono_arch_create_llvm_native_thunk (mono_domain_get (), (guint8*)(*symbol));
6816 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
6818 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
6822 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
6824 LLVMTypeRef param_types [4];
6826 param_types [0] = param_type1;
6827 param_types [1] = param_type2;
6829 AddFunc (module, name, ret_type, param_types, 2);
6833 add_intrinsics (LLVMModuleRef module)
6835 /* Emit declarations of instrinsics */
6837 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
6838 * type doesn't seem to do any locking.
6841 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
6843 memset_param_count = 5;
6844 memset_func_name = "llvm.memset.p0i8.i32";
6846 AddFunc (module, memset_func_name, LLVMVoidType (), params, memset_param_count);
6850 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
6852 memcpy_param_count = 5;
6853 memcpy_func_name = "llvm.memcpy.p0i8.p0i8.i32";
6855 AddFunc (module, memcpy_func_name, LLVMVoidType (), params, memcpy_param_count);
6859 LLVMTypeRef params [] = { LLVMDoubleType () };
6861 AddFunc (module, "llvm.sin.f64", LLVMDoubleType (), params, 1);
6862 AddFunc (module, "llvm.cos.f64", LLVMDoubleType (), params, 1);
6863 AddFunc (module, "llvm.sqrt.f64", LLVMDoubleType (), params, 1);
6865 /* This isn't an intrinsic, instead llvm seems to special case it by name */
6866 AddFunc (module, "fabs", LLVMDoubleType (), params, 1);
6870 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
6871 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
6872 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
6874 AddFunc (module, "llvm.sadd.with.overflow.i32", ret_type, params, 2);
6875 AddFunc (module, "llvm.sadd.with.overflow.i32", ret_type, params, 2);
6876 AddFunc (module, "llvm.uadd.with.overflow.i32", ret_type, params, 2);
6877 AddFunc (module, "llvm.ssub.with.overflow.i32", ret_type, params, 2);
6878 AddFunc (module, "llvm.usub.with.overflow.i32", ret_type, params, 2);
6879 AddFunc (module, "llvm.smul.with.overflow.i32", ret_type, params, 2);
6880 AddFunc (module, "llvm.umul.with.overflow.i32", ret_type, params, 2);
6884 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
6885 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
6886 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
6888 AddFunc (module, "llvm.sadd.with.overflow.i64", ret_type, params, 2);
6889 AddFunc (module, "llvm.uadd.with.overflow.i64", ret_type, params, 2);
6890 AddFunc (module, "llvm.ssub.with.overflow.i64", ret_type, params, 2);
6891 AddFunc (module, "llvm.usub.with.overflow.i64", ret_type, params, 2);
6892 AddFunc (module, "llvm.smul.with.overflow.i64", ret_type, params, 2);
6893 AddFunc (module, "llvm.umul.with.overflow.i64", ret_type, params, 2);
6896 AddFunc2 (module, "llvm.expect.i8", LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
6900 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
6902 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
6905 /* SSE intrinsics */
6906 #if defined(TARGET_X86) || defined(TARGET_AMD64)
6908 LLVMTypeRef ret_type, arg_types [16];
6911 ret_type = type_to_simd_type (MONO_TYPE_I4);
6912 arg_types [0] = ret_type;
6913 arg_types [1] = ret_type;
6914 AddFunc (module, "llvm.x86.sse41.pminud", ret_type, arg_types, 2);
6915 AddFunc (module, "llvm.x86.sse41.pmaxud", ret_type, arg_types, 2);
6917 ret_type = type_to_simd_type (MONO_TYPE_I2);
6918 arg_types [0] = ret_type;
6919 arg_types [1] = ret_type;
6920 AddFunc (module, "llvm.x86.sse41.pminuw", ret_type, arg_types, 2);
6921 AddFunc (module, "llvm.x86.sse2.pmins.w", ret_type, arg_types, 2);
6922 AddFunc (module, "llvm.x86.sse41.pmaxuw", ret_type, arg_types, 2);
6923 AddFunc (module, "llvm.x86.sse2.padds.w", ret_type, arg_types, 2);
6924 AddFunc (module, "llvm.x86.sse2.psubs.w", ret_type, arg_types, 2);
6925 AddFunc (module, "llvm.x86.sse2.paddus.w", ret_type, arg_types, 2);
6926 AddFunc (module, "llvm.x86.sse2.psubus.w", ret_type, arg_types, 2);
6927 AddFunc (module, "llvm.x86.sse2.pavg.w", ret_type, arg_types, 2);
6928 AddFunc (module, "llvm.x86.sse2.pmulh.w", ret_type, arg_types, 2);
6929 AddFunc (module, "llvm.x86.sse2.pmulhu.w", ret_type, arg_types, 2);
6931 ret_type = type_to_simd_type (MONO_TYPE_I1);
6932 arg_types [0] = ret_type;
6933 arg_types [1] = ret_type;
6934 AddFunc (module, "llvm.x86.sse2.pminu.b", ret_type, arg_types, 2);
6935 AddFunc (module, "llvm.x86.sse2.pmaxu.b", ret_type, arg_types, 2);
6936 AddFunc (module, "llvm.x86.sse2.padds.b", ret_type, arg_types, 2);
6937 AddFunc (module, "llvm.x86.sse2.psubs.b", ret_type, arg_types, 2);
6938 AddFunc (module, "llvm.x86.sse2.paddus.b", ret_type, arg_types, 2);
6939 AddFunc (module, "llvm.x86.sse2.psubus.b", ret_type, arg_types, 2);
6940 AddFunc (module, "llvm.x86.sse2.pavg.b", ret_type, arg_types, 2);
6942 ret_type = type_to_simd_type (MONO_TYPE_R8);
6943 arg_types [0] = ret_type;
6944 arg_types [1] = ret_type;
6945 AddFunc (module, "llvm.x86.sse2.min.pd", ret_type, arg_types, 2);
6946 AddFunc (module, "llvm.x86.sse2.max.pd", ret_type, arg_types, 2);
6947 AddFunc (module, "llvm.x86.sse3.hadd.pd", ret_type, arg_types, 2);
6948 AddFunc (module, "llvm.x86.sse3.hsub.pd", ret_type, arg_types, 2);
6949 AddFunc (module, "llvm.x86.sse3.addsub.pd", ret_type, arg_types, 2);
6951 ret_type = type_to_simd_type (MONO_TYPE_R4);
6952 arg_types [0] = ret_type;
6953 arg_types [1] = ret_type;
6954 AddFunc (module, "llvm.x86.sse.min.ps", ret_type, arg_types, 2);
6955 AddFunc (module, "llvm.x86.sse.max.ps", ret_type, arg_types, 2);
6956 AddFunc (module, "llvm.x86.sse3.hadd.ps", ret_type, arg_types, 2);
6957 AddFunc (module, "llvm.x86.sse3.hsub.ps", ret_type, arg_types, 2);
6958 AddFunc (module, "llvm.x86.sse3.addsub.ps", ret_type, arg_types, 2);
6961 ret_type = type_to_simd_type (MONO_TYPE_I1);
6962 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
6963 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
6964 AddFunc (module, "llvm.x86.sse2.packsswb.128", ret_type, arg_types, 2);
6965 AddFunc (module, "llvm.x86.sse2.packuswb.128", ret_type, arg_types, 2);
6966 ret_type = type_to_simd_type (MONO_TYPE_I2);
6967 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
6968 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
6969 AddFunc (module, "llvm.x86.sse2.packssdw.128", ret_type, arg_types, 2);
6970 AddFunc (module, "llvm.x86.sse41.packusdw", ret_type, arg_types, 2);
6973 ret_type = type_to_simd_type (MONO_TYPE_R8);
6974 arg_types [0] = ret_type;
6975 arg_types [1] = ret_type;
6976 arg_types [2] = LLVMInt8Type ();
6977 AddFunc (module, "llvm.x86.sse2.cmp.pd", ret_type, arg_types, 3);
6978 ret_type = type_to_simd_type (MONO_TYPE_R4);
6979 arg_types [0] = ret_type;
6980 arg_types [1] = ret_type;
6981 arg_types [2] = LLVMInt8Type ();
6982 AddFunc (module, "llvm.x86.sse.cmp.ps", ret_type, arg_types, 3);
6984 /* Conversion ops */
6985 ret_type = type_to_simd_type (MONO_TYPE_R8);
6986 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
6987 AddFunc (module, "llvm.x86.sse2.cvtdq2pd", ret_type, arg_types, 1);
6988 ret_type = type_to_simd_type (MONO_TYPE_R4);
6989 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
6990 AddFunc (module, "llvm.x86.sse2.cvtdq2ps", ret_type, arg_types, 1);
6991 ret_type = type_to_simd_type (MONO_TYPE_I4);
6992 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
6993 AddFunc (module, "llvm.x86.sse2.cvtpd2dq", ret_type, arg_types, 1);
6994 ret_type = type_to_simd_type (MONO_TYPE_I4);
6995 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
6996 AddFunc (module, "llvm.x86.sse2.cvtps2dq", ret_type, arg_types, 1);
6997 ret_type = type_to_simd_type (MONO_TYPE_R4);
6998 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
6999 AddFunc (module, "llvm.x86.sse2.cvtpd2ps", ret_type, arg_types, 1);
7000 ret_type = type_to_simd_type (MONO_TYPE_R8);
7001 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7002 AddFunc (module, "llvm.x86.sse2.cvtps2pd", ret_type, arg_types, 1);
7004 ret_type = type_to_simd_type (MONO_TYPE_I4);
7005 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7006 AddFunc (module, "llvm.x86.sse2.cvttpd2dq", ret_type, arg_types, 1);
7007 ret_type = type_to_simd_type (MONO_TYPE_I4);
7008 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7009 AddFunc (module, "llvm.x86.sse2.cvttps2dq", ret_type, arg_types, 1);
7012 ret_type = type_to_simd_type (MONO_TYPE_R8);
7013 arg_types [0] = ret_type;
7014 AddFunc (module, "llvm.x86.sse2.sqrt.pd", ret_type, arg_types, 1);
7015 ret_type = type_to_simd_type (MONO_TYPE_R4);
7016 arg_types [0] = ret_type;
7017 AddFunc (module, "llvm.x86.sse.sqrt.ps", ret_type, arg_types, 1);
7018 ret_type = type_to_simd_type (MONO_TYPE_R4);
7019 arg_types [0] = ret_type;
7020 AddFunc (module, "llvm.x86.sse.rsqrt.ps", ret_type, arg_types, 1);
7021 ret_type = type_to_simd_type (MONO_TYPE_R4);
7022 arg_types [0] = ret_type;
7023 AddFunc (module, "llvm.x86.sse.rcp.ps", ret_type, arg_types, 1);
7026 ret_type = type_to_simd_type (MONO_TYPE_I2);
7027 arg_types [0] = ret_type;
7028 arg_types [1] = LLVMInt32Type ();
7029 AddFunc (module, "llvm.x86.sse2.psrli.w", ret_type, arg_types, 2);
7030 AddFunc (module, "llvm.x86.sse2.psrai.w", ret_type, arg_types, 2);
7031 AddFunc (module, "llvm.x86.sse2.pslli.w", ret_type, arg_types, 2);
7032 ret_type = type_to_simd_type (MONO_TYPE_I4);
7033 arg_types [0] = ret_type;
7034 arg_types [1] = LLVMInt32Type ();
7035 AddFunc (module, "llvm.x86.sse2.psrli.d", ret_type, arg_types, 2);
7036 AddFunc (module, "llvm.x86.sse2.psrai.d", ret_type, arg_types, 2);
7037 AddFunc (module, "llvm.x86.sse2.pslli.d", ret_type, arg_types, 2);
7038 ret_type = type_to_simd_type (MONO_TYPE_I8);
7039 arg_types [0] = ret_type;
7040 arg_types [1] = LLVMInt32Type ();
7041 AddFunc (module, "llvm.x86.sse2.psrli.q", ret_type, arg_types, 2);
7042 AddFunc (module, "llvm.x86.sse2.pslli.q", ret_type, arg_types, 2);
7045 ret_type = LLVMInt32Type ();
7046 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
7047 AddFunc (module, "llvm.x86.sse2.pmovmskb.128", ret_type, arg_types, 1);
7050 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
7053 /* Load/Store intrinsics */
7055 LLVMTypeRef arg_types [5];
7059 for (i = 1; i <= 8; i *= 2) {
7060 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
7061 arg_types [1] = LLVMInt32Type ();
7062 arg_types [2] = LLVMInt1Type ();
7063 arg_types [3] = LLVMInt32Type ();
7064 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
7065 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
7067 arg_types [0] = LLVMIntType (i * 8);
7068 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
7069 arg_types [2] = LLVMInt32Type ();
7070 arg_types [3] = LLVMInt1Type ();
7071 arg_types [4] = LLVMInt32Type ();
7072 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
7073 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
7079 add_types (MonoLLVMModule *module)
7081 module->ptr_type = LLVMPointerType (sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
7085 mono_llvm_init (void)
7087 mono_native_tls_alloc (¤t_cfg_tls_id, NULL);
7091 init_jit_module (MonoDomain *domain)
7093 MonoJitICallInfo *info;
7094 MonoJitDomainInfo *dinfo;
7095 MonoLLVMModule *module;
7098 dinfo = domain_jit_info (domain);
7099 if (dinfo->llvm_module)
7102 mono_loader_lock ();
7104 if (dinfo->llvm_module) {
7105 mono_loader_unlock ();
7109 module = g_new0 (MonoLLVMModule, 1);
7111 name = g_strdup_printf ("mono-%s", domain->friendly_name);
7112 module->lmodule = LLVMModuleCreateWithName (name);
7113 module->context = LLVMGetGlobalContext ();
7115 module->mono_ee = mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->lmodule), alloc_cb, emitted_cb, exception_cb, dlsym_cb, &module->ee);
7117 add_intrinsics (module->lmodule);
7120 module->llvm_types = g_hash_table_new (NULL, NULL);
7122 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
7124 LLVMAddGlobalMapping (module->ee, LLVMGetNamedFunction (module->lmodule, "llvm_resume_unwind_trampoline"), (void*)info->func);
7126 mono_memory_barrier ();
7128 dinfo->llvm_module = module;
7130 mono_loader_unlock ();
7134 mono_llvm_cleanup (void)
7136 MonoLLVMModule *module = &aot_module;
7138 if (module->lmodule)
7139 LLVMDisposeModule (module->lmodule);
7141 if (module->context)
7142 LLVMContextDispose (module->context);
7146 mono_llvm_free_domain_info (MonoDomain *domain)
7148 MonoJitDomainInfo *info = domain_jit_info (domain);
7149 MonoLLVMModule *module = info->llvm_module;
7155 if (module->llvm_types)
7156 g_hash_table_destroy (module->llvm_types);
7158 mono_llvm_dispose_ee (module->mono_ee);
7160 if (module->bb_names) {
7161 for (i = 0; i < module->bb_names_len; ++i)
7162 g_free (module->bb_names [i]);
7163 g_free (module->bb_names);
7165 //LLVMDisposeModule (module->module);
7169 info->llvm_module = NULL;
7173 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, gboolean emit_dwarf, gboolean static_link, gboolean llvm_only)
7175 MonoLLVMModule *module = &aot_module;
7177 /* Delete previous module */
7178 if (module->plt_entries)
7179 g_hash_table_destroy (module->plt_entries);
7180 if (module->lmodule)
7181 LLVMDisposeModule (module->lmodule);
7183 memset (module, 0, sizeof (aot_module));
7185 module->lmodule = LLVMModuleCreateWithName ("aot");
7186 module->assembly = assembly;
7187 module->global_prefix = g_strdup (global_prefix);
7188 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
7189 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
7190 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
7191 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
7192 module->external_symbols = TRUE;
7193 module->emit_dwarf = emit_dwarf;
7194 module->static_link = static_link;
7195 module->llvm_only = llvm_only;
7196 /* The first few entries are reserved */
7197 module->max_got_offset = 16;
7198 module->context = LLVMContextCreate ();
7200 add_intrinsics (module->lmodule);
7205 * We couldn't compute the type of the LLVM global representing the got because
7206 * its size is only known after all the methods have been emitted. So create
7207 * a dummy variable, and replace all uses it with the real got variable when
7208 * its size is known in mono_llvm_emit_aot_module ().
7211 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
7213 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
7214 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
7217 /* Add initialization array */
7219 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
7221 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
7222 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
7226 emit_init_icall_wrappers (module);
7228 emit_llvm_code_start (module);
7230 /* Add a dummy personality function */
7231 if (!use_debug_personality) {
7232 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
7233 LLVMSetLinkage (personality, LLVMExternalLinkage);
7234 mark_as_used (module, personality);
7237 /* Add a reference to the c++ exception we throw/catch */
7239 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
7240 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
7241 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
7242 mono_llvm_set_is_constant (module->sentinel_exception);
7245 module->llvm_types = g_hash_table_new (NULL, NULL);
7246 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
7247 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
7248 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
7249 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
7250 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
7251 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
7252 module->method_to_callers = g_hash_table_new (NULL, NULL);
7256 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
7259 LLVMValueRef res, *vals;
7261 vals = g_new0 (LLVMValueRef, nvalues);
7262 for (i = 0; i < nvalues; ++i)
7263 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
7264 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
7270 * mono_llvm_emit_aot_file_info:
7272 * Emit the MonoAotFileInfo structure.
7273 * Same as emit_aot_file_info () in aot-compiler.c.
7276 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
7278 MonoLLVMModule *module = &aot_module;
7280 /* Save these for later */
7281 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
7282 module->has_jitted_code = has_jitted_code;
7286 * mono_llvm_emit_aot_data:
7288 * Emit the binary data DATA pointed to by symbol SYMBOL.
7291 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
7293 MonoLLVMModule *module = &aot_module;
7297 type = LLVMArrayType (LLVMInt8Type (), data_len);
7298 d = LLVMAddGlobal (module->lmodule, type, symbol);
7299 LLVMSetVisibility (d, LLVMHiddenVisibility);
7300 LLVMSetLinkage (d, LLVMInternalLinkage);
7301 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
7302 mono_llvm_set_is_constant (d);
7305 /* Add a reference to a global defined in JITted code */
7307 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
7312 s = g_strdup_printf ("%s%s", module->global_prefix, name);
7313 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
7319 emit_aot_file_info (MonoLLVMModule *module)
7321 LLVMTypeRef file_info_type;
7322 LLVMTypeRef *eltypes, eltype;
7323 LLVMValueRef info_var;
7324 LLVMValueRef *fields;
7325 int i, nfields, tindex;
7326 MonoAotFileInfo *info;
7327 LLVMModuleRef lmodule = module->lmodule;
7329 info = &module->aot_info;
7331 /* Create an LLVM type to represent MonoAotFileInfo */
7332 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 14 + 4;
7333 eltypes = g_new (LLVMTypeRef, nfields);
7335 eltypes [tindex ++] = LLVMInt32Type ();
7336 eltypes [tindex ++] = LLVMInt32Type ();
7338 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
7339 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
7341 for (i = 0; i < 14; ++i)
7342 eltypes [tindex ++] = LLVMInt32Type ();
7344 for (i = 0; i < 4; ++i)
7345 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
7346 g_assert (tindex == nfields);
7347 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
7348 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
7350 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
7351 if (module->static_link) {
7352 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
7353 LLVMSetLinkage (info_var, LLVMInternalLinkage);
7355 fields = g_new (LLVMValueRef, nfields);
7357 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
7358 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
7362 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
7363 * for symbols defined in the .s file emitted by the aot compiler.
7365 eltype = eltypes [tindex];
7366 if (module->llvm_only)
7367 fields [tindex ++] = LLVMConstNull (eltype);
7369 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
7370 fields [tindex ++] = module->got_var;
7371 /* llc defines this directly */
7372 if (!module->llvm_only) {
7373 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
7374 fields [tindex ++] = LLVMConstNull (eltype);
7375 fields [tindex ++] = LLVMConstNull (eltype);
7377 fields [tindex ++] = LLVMConstNull (eltype);
7378 fields [tindex ++] = module->get_method;
7379 fields [tindex ++] = module->get_unbox_tramp;
7381 if (module->has_jitted_code) {
7382 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
7383 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
7385 fields [tindex ++] = LLVMConstNull (eltype);
7386 fields [tindex ++] = LLVMConstNull (eltype);
7388 if (!module->llvm_only)
7389 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
7391 fields [tindex ++] = LLVMConstNull (eltype);
7392 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
7393 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
7394 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
7395 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
7396 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
7397 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
7398 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
7399 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
7400 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
7401 /* Not needed (mem_end) */
7402 fields [tindex ++] = LLVMConstNull (eltype);
7403 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
7404 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
7405 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
7406 if (info->trampoline_size [0]) {
7407 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
7408 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
7409 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_thunks");
7410 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
7412 fields [tindex ++] = LLVMConstNull (eltype);
7413 fields [tindex ++] = LLVMConstNull (eltype);
7414 fields [tindex ++] = LLVMConstNull (eltype);
7415 fields [tindex ++] = LLVMConstNull (eltype);
7417 if (module->static_link)
7418 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
7420 fields [tindex ++] = LLVMConstNull (eltype);
7421 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
7422 if (!module->llvm_only) {
7423 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
7424 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
7425 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
7426 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
7427 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
7428 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
7430 fields [tindex ++] = LLVMConstNull (eltype);
7431 fields [tindex ++] = LLVMConstNull (eltype);
7432 fields [tindex ++] = LLVMConstNull (eltype);
7433 fields [tindex ++] = LLVMConstNull (eltype);
7434 fields [tindex ++] = LLVMConstNull (eltype);
7435 fields [tindex ++] = LLVMConstNull (eltype);
7438 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
7439 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
7442 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
7443 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
7444 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
7445 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
7446 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
7447 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
7448 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
7449 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
7450 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
7451 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
7452 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
7453 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
7454 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
7455 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
7457 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
7458 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
7459 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
7460 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
7461 g_assert (tindex == nfields);
7463 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
7465 if (module->static_link) {
7469 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
7470 /* Get rid of characters which cannot occur in symbols */
7472 for (p = s; *p; ++p) {
7473 if (!(isalnum (*p) || *p == '_'))
7476 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
7478 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
7479 LLVMSetLinkage (var, LLVMExternalLinkage);
7484 * Emit the aot module into the LLVM bitcode file FILENAME.
7487 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
7489 LLVMTypeRef got_type, inited_type;
7490 LLVMValueRef real_got, real_inited;
7491 MonoLLVMModule *module = &aot_module;
7493 emit_llvm_code_end (module);
7496 * Create the real got variable and replace all uses of the dummy variable with
7499 got_type = LLVMArrayType (module->ptr_type, module->max_got_offset + 1);
7500 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
7501 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
7502 if (module->external_symbols) {
7503 LLVMSetLinkage (real_got, LLVMExternalLinkage);
7504 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
7506 LLVMSetLinkage (real_got, LLVMInternalLinkage);
7508 mono_llvm_replace_uses_of (module->got_var, real_got);
7510 mark_as_used (&aot_module, real_got);
7512 /* Delete the dummy got so it doesn't become a global */
7513 LLVMDeleteGlobal (module->got_var);
7514 module->got_var = real_got;
7517 * Same for the init_var
7519 if (module->llvm_only) {
7520 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
7521 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
7522 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
7523 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
7524 mono_llvm_replace_uses_of (module->inited_var, real_inited);
7525 LLVMDeleteGlobal (module->inited_var);
7528 if (module->llvm_only) {
7529 emit_get_method (&aot_module);
7530 emit_get_unbox_tramp (&aot_module);
7533 emit_llvm_used (&aot_module);
7534 emit_dbg_info (&aot_module, filename, cu_name);
7535 emit_aot_file_info (&aot_module);
7538 * Replace GOT entries for directly callable methods with the methods themselves.
7539 * It would be easier to implement this by predefining all methods before compiling
7540 * their bodies, but that couldn't handle the case when a method fails to compile
7543 if (module->llvm_only) {
7544 GHashTableIter iter;
7546 GSList *callers, *l;
7548 g_hash_table_iter_init (&iter, module->method_to_callers);
7549 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
7550 LLVMValueRef lmethod;
7552 lmethod = g_hash_table_lookup (module->method_to_lmethod, method);
7554 for (l = callers; l; l = l->next) {
7555 LLVMValueRef caller = l->data;
7557 mono_llvm_replace_uses_of (caller, lmethod);
7563 /* Replace PLT entries for directly callable methods with the methods themselves */
7565 GHashTableIter iter;
7567 LLVMValueRef callee;
7569 g_hash_table_iter_init (&iter, module->plt_entries_ji);
7570 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
7571 if (mono_aot_is_direct_callable (ji)) {
7572 LLVMValueRef lmethod;
7574 lmethod = g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
7575 /* The types might not match because the caller might pass an rgctx */
7576 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
7577 mono_llvm_replace_uses_of (callee, lmethod);
7578 mono_aot_mark_unused_llvm_plt_entry (ji);
7588 if (LLVMVerifyModule (module->module, LLVMReturnStatusAction, &verifier_err)) {
7589 g_assert_not_reached ();
7594 LLVMWriteBitcodeToFile (module->lmodule, filename);
7599 md_string (const char *s)
7601 return LLVMMDString (s, strlen (s));
7604 /* Debugging support */
7607 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
7609 LLVMModuleRef lmodule = module->lmodule;
7610 LLVMValueRef args [16], cu_args [16], cu, ver;
7612 char *build_info, *s, *dir;
7615 * This can only be enabled when LLVM code is emitted into a separate object
7616 * file, since the AOT compiler also emits dwarf info,
7617 * and the abbrev indexes will not be correct since llvm has added its own
7620 if (!module->emit_dwarf)
7624 * Emit dwarf info in the form of LLVM metadata. There is some
7625 * out-of-date documentation at:
7626 * http://llvm.org/docs/SourceLevelDebugging.html
7627 * but most of this was gathered from the llvm and
7632 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_TAG_compile_unit, FALSE);
7633 /* CU name/compilation dir */
7634 dir = g_path_get_dirname (filename);
7635 args [0] = LLVMMDString (cu_name, strlen (cu_name));
7636 args [1] = LLVMMDString (dir, strlen (dir));
7637 cu_args [n_cuargs ++] = LLVMMDNode (args, 2);
7640 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_LANG_C99, FALSE);
7642 build_info = mono_get_runtime_build_info ();
7643 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
7644 cu_args [n_cuargs ++] = LLVMMDString (s, strlen (s));
7645 g_free (build_info);
7647 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7649 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
7650 /* Runtime version */
7651 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7653 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
7654 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
7656 if (module->subprogram_mds) {
7660 mds = g_new0 (LLVMValueRef, module->subprogram_mds->len);
7661 for (i = 0; i < module->subprogram_mds->len; ++i)
7662 mds [i] = g_ptr_array_index (module->subprogram_mds, i);
7663 cu_args [n_cuargs ++] = LLVMMDNode (mds, module->subprogram_mds->len);
7665 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
7668 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
7669 /* Imported modules */
7670 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
7672 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
7673 /* DebugEmissionKind = FullDebug */
7674 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7675 cu = LLVMMDNode (cu_args, n_cuargs);
7676 LLVMAddNamedMetadataOperand (lmodule, "llvm.dbg.cu", cu);
7678 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7679 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
7680 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
7681 ver = LLVMMDNode (args, 3);
7682 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
7684 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7685 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
7686 args [2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7687 ver = LLVMMDNode (args, 3);
7688 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
7692 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
7694 MonoLLVMModule *module = ctx->module;
7695 MonoDebugMethodInfo *minfo = ctx->minfo;
7696 char *source_file, *dir, *filename;
7697 LLVMValueRef md, args [16], ctx_args [16], md_args [64], type_args [16], ctx_md, type_md;
7698 MonoSymSeqPoint *sym_seq_points;
7704 mono_debug_symfile_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
7706 source_file = g_strdup ("<unknown>");
7707 dir = g_path_get_dirname (source_file);
7708 filename = g_path_get_basename (source_file);
7710 ctx_args [0] = LLVMConstInt (LLVMInt32Type (), 0x29, FALSE);
7711 args [0] = md_string (filename);
7712 args [1] = md_string (dir);
7713 ctx_args [1] = LLVMMDNode (args, 2);
7714 ctx_md = LLVMMDNode (ctx_args, 2);
7716 type_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subroutine_type, FALSE);
7717 type_args [1] = NULL;
7718 type_args [2] = NULL;
7719 type_args [3] = LLVMMDString ("", 0);
7720 type_args [4] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7721 type_args [5] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
7722 type_args [6] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
7723 type_args [7] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
7724 type_args [8] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7725 type_args [9] = NULL;
7726 type_args [10] = NULL;
7727 type_args [11] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7728 type_args [12] = NULL;
7729 type_args [13] = NULL;
7730 type_args [14] = NULL;
7731 type_md = LLVMMDNode (type_args, 14);
7733 /* http://llvm.org/docs/SourceLevelDebugging.html#subprogram-descriptors */
7734 md_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subprogram, FALSE);
7735 /* Source directory + file pair */
7736 args [0] = md_string (filename);
7737 args [1] = md_string (dir);
7738 md_args [1] = LLVMMDNode (args ,2);
7739 md_args [2] = ctx_md;
7740 md_args [3] = md_string (cfg->method->name);
7741 md_args [4] = md_string (name);
7742 md_args [5] = md_string (name);
7745 md_args [6] = LLVMConstInt (LLVMInt32Type (), sym_seq_points [0].line, FALSE);
7747 md_args [6] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7749 md_args [7] = type_md;
7751 md_args [8] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
7753 md_args [9] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
7755 md_args [10] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
7756 /* Index into a virtual function */
7757 md_args [11] = NULL;
7758 md_args [12] = NULL;
7760 md_args [13] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
7762 md_args [14] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
7763 /* Pointer to LLVM function */
7764 md_args [15] = method;
7765 /* Function template parameter */
7766 md_args [16] = NULL;
7767 /* Function declaration descriptor */
7768 md_args [17] = NULL;
7769 /* List of function variables */
7770 md_args [18] = LLVMMDNode (args, 0);
7772 md_args [19] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
7773 md = LLVMMDNode (md_args, 20);
7775 if (!module->subprogram_mds)
7776 module->subprogram_mds = g_ptr_array_new ();
7777 g_ptr_array_add (module->subprogram_mds, md);
7781 g_free (source_file);
7782 g_free (sym_seq_points);
7788 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
7790 MonoCompile *cfg = ctx->cfg;
7792 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
7793 MonoDebugSourceLocation *loc;
7794 LLVMValueRef loc_md, md_args [16];
7797 loc = mono_debug_symfile_lookup_location (ctx->minfo, cil_code - cfg->header->code);
7801 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->row, FALSE);
7802 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->column, FALSE);
7803 md_args [nmd_args ++] = ctx->dbg_md;
7804 md_args [nmd_args ++] = NULL;
7805 loc_md = LLVMMDNode (md_args, nmd_args);
7806 LLVMSetCurrentDebugLocation (builder, loc_md);
7807 mono_debug_symfile_free_location (loc);
7813 default_mono_llvm_unhandled_exception (void)
7815 MonoJitTlsData *jit_tls = mono_get_jit_tls ();
7816 MonoObject *target = mono_gchandle_get_target (jit_tls->thrown_exc);
7818 mono_unhandled_exception (target);
7819 exit (mono_environment_exitcode_get ());
7824 - Emit LLVM IR from the mono IR using the LLVM C API.
7825 - The original arch specific code remains, so we can fall back to it if we run
7826 into something we can't handle.
7830 A partial list of issues:
7831 - Handling of opcodes which can throw exceptions.
7833 In the mono JIT, these are implemented using code like this:
7840 push throw_pos - method
7841 call <exception trampoline>
7843 The problematic part is push throw_pos - method, which cannot be represented
7844 in the LLVM IR, since it does not support label values.
7845 -> this can be implemented in AOT mode using inline asm + labels, but cannot
7846 be implemented in JIT mode ?
7847 -> a possible but slower implementation would use the normal exception
7848 throwing code but it would need to control the placement of the throw code
7849 (it needs to be exactly after the compare+branch).
7850 -> perhaps add a PC offset intrinsics ?
7852 - efficient implementation of .ovf opcodes.
7854 These are currently implemented as:
7855 <ins which sets the condition codes>
7858 Some overflow opcodes are now supported by LLVM SVN.
7860 - exception handling, unwinding.
7861 - SSA is disabled for methods with exception handlers
7862 - How to obtain unwind info for LLVM compiled methods ?
7863 -> this is now solved by converting the unwind info generated by LLVM
7865 - LLVM uses the c++ exception handling framework, while we use our home grown
7866 code, and couldn't use the c++ one:
7867 - its not supported under VC++, other exotic platforms.
7868 - it might be impossible to support filter clauses with it.
7872 The trampolines need a predictable call sequence, since they need to disasm
7873 the calling code to obtain register numbers / offsets.
7875 LLVM currently generates this code in non-JIT mode:
7876 mov -0x98(%rax),%eax
7878 Here, the vtable pointer is lost.
7879 -> solution: use one vtable trampoline per class.
7881 - passing/receiving the IMT pointer/RGCTX.
7882 -> solution: pass them as normal arguments ?
7886 LLVM does not allow the specification of argument registers etc. This means
7887 that all calls are made according to the platform ABI.
7889 - passing/receiving vtypes.
7891 Vtypes passed/received in registers are handled by the front end by using
7892 a signature with scalar arguments, and loading the parts of the vtype into those
7895 Vtypes passed on the stack are handled using the 'byval' attribute.
7899 Supported though alloca, we need to emit the load/store code.
7903 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
7904 typed registers, so we have to keep track of the precise LLVM type of each vreg.
7905 This is made easier because the IR is already in SSA form.
7906 An additional problem is that our IR is not consistent with types, i.e. i32/ia64
7907 types are frequently used incorrectly.
7912 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
7913 it with the file containing the methods emitted by the JIT and the AOT data
7917 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
7918 * - each bblock should end with a branch
7919 * - setting the return value, making cfg->ret non-volatile
7920 * - avoid some transformations in the JIT which make it harder for us to generate
7922 * - use pointer types to help optimizations.