2 * mini-llvm.c: llvm "Backend" for the mono JIT
4 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
5 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
9 #include <mono/metadata/debug-helpers.h>
10 #include <mono/metadata/debug-mono-symfile.h>
11 #include <mono/metadata/mempool-internals.h>
12 #include <mono/metadata/environment.h>
13 #include <mono/metadata/object-internals.h>
14 #include <mono/metadata/abi-details.h>
15 #include <mono/utils/mono-tls.h>
16 #include <mono/utils/mono-dl.h>
17 #include <mono/utils/mono-time.h>
18 #include <mono/utils/freebsd-dwarf.h>
20 #ifndef __STDC_LIMIT_MACROS
21 #define __STDC_LIMIT_MACROS
23 #ifndef __STDC_CONSTANT_MACROS
24 #define __STDC_CONSTANT_MACROS
27 #include "llvm-c/BitWriter.h"
28 #include "llvm-c/Analysis.h"
30 #include "mini-llvm-cpp.h"
32 #include "aot-compiler.h"
33 #include "mini-llvm.h"
38 extern void *memset(void *, int, size_t);
39 void bzero (void *to, size_t count) { memset (to, 0, count); }
43 #if LLVM_API_VERSION < 4
44 #error "The version of the mono llvm repository is too old."
47 #define ALIGN_PTR_TO(ptr,align) (gpointer)((((gssize)(ptr)) + (align - 1)) & (~(align - 1)))
50 * Information associated by mono with LLVM modules.
53 LLVMModuleRef lmodule;
54 LLVMValueRef throw_icall, rethrow, match_exc, throw_corlib_exception, resume_eh;
55 GHashTable *llvm_types;
57 const char *got_symbol;
58 const char *get_method_symbol;
59 const char *get_unbox_tramp_symbol;
60 GHashTable *plt_entries;
61 GHashTable *plt_entries_ji;
62 GHashTable *method_to_lmethod;
63 GHashTable *direct_callables;
68 GPtrArray *subprogram_mds;
70 LLVMExecutionEngineRef ee;
71 gboolean external_symbols;
76 MonoAssembly *assembly;
78 MonoAotFileInfo aot_info;
79 const char *jit_got_symbol;
80 const char *eh_frame_symbol;
81 LLVMValueRef get_method, get_unbox_tramp;
82 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
83 LLVMValueRef code_start, code_end;
84 LLVMValueRef inited_var;
85 int max_inited_idx, max_method_idx;
86 gboolean has_jitted_code;
89 GHashTable *idx_to_lmethod;
90 GHashTable *idx_to_unbox_tramp;
91 /* Maps a MonoMethod to LLVM instructions representing it */
92 GHashTable *method_to_callers;
93 LLVMContextRef context;
94 LLVMValueRef sentinel_exception;
95 void *di_builder, *cu;
99 * Information associated by the backend with mono basic blocks.
102 LLVMBasicBlockRef bblock, end_bblock;
103 LLVMValueRef finally_ind;
104 gboolean added, invoke_target;
106 * If this bblock is the start of a finally clause, this is a list of bblocks it
107 * needs to branch to in ENDFINALLY.
109 GSList *call_handler_return_bbs;
111 * If this bblock is the start of a finally clause, this is the bblock that
112 * CALL_HANDLER needs to branch to.
114 LLVMBasicBlockRef call_handler_target_bb;
115 /* The list of switch statements generated by ENDFINALLY instructions */
116 GSList *endfinally_switch_ins_list;
121 * Structure containing emit state
124 MonoMemPool *mempool;
126 /* Maps method names to the corresponding LLVMValueRef */
127 GHashTable *emitted_method_decls;
130 LLVMValueRef lmethod;
131 MonoLLVMModule *module;
132 LLVMModuleRef lmodule;
134 int sindex, default_index, ex_index;
135 LLVMBuilderRef builder;
136 LLVMValueRef *values, *addresses;
137 MonoType **vreg_cli_types;
139 MonoMethodSignature *sig;
141 GHashTable *region_to_handler;
142 GHashTable *clause_to_handler;
143 LLVMBuilderRef alloca_builder;
144 LLVMValueRef last_alloca;
145 LLVMValueRef rgctx_arg;
146 LLVMValueRef this_arg;
147 LLVMTypeRef *vreg_types;
148 LLVMTypeRef method_type;
149 LLVMBasicBlockRef init_bb, inited_bb;
151 gboolean *unreachable;
153 gboolean has_got_access;
154 gboolean is_linkonce;
155 int this_arg_pindex, rgctx_arg_pindex;
156 LLVMValueRef imt_rgctx_loc;
157 GHashTable *llvm_types;
159 MonoDebugMethodInfo *minfo;
161 /* For every clause, the clauses it is nested in */
164 GHashTable *exc_meta;
165 GHashTable *method_to_callers;
166 GPtrArray *phi_values;
167 GPtrArray *bblock_list;
169 GHashTable *jit_callees;
175 MonoBasicBlock *in_bb;
180 * Instruction metadata
181 * This is the same as ins_info, but LREG != IREG.
189 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
190 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
197 /* keep in sync with the enum in mini.h */
200 #include "mini-ops.h"
205 #if SIZEOF_VOID_P == 4
206 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
208 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
211 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
214 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
216 #define TRACE_FAILURE(msg)
220 #define IS_TARGET_X86 1
222 #define IS_TARGET_X86 0
226 #define IS_TARGET_AMD64 1
228 #define IS_TARGET_AMD64 0
231 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
233 static LLVMIntPredicate cond_to_llvm_cond [] = {
246 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
259 static MonoNativeTlsKey current_cfg_tls_id;
261 static MonoLLVMModule aot_module;
263 static GHashTable *intrins_id_to_name;
264 static GHashTable *intrins_name_to_id;
266 static void init_jit_module (MonoDomain *domain);
268 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
269 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
270 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
271 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
272 static LLVMValueRef get_intrinsic (EmitContext *ctx, const char *name);
273 static void decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame);
276 set_failure (EmitContext *ctx, const char *message)
278 TRACE_FAILURE (reason);
279 ctx->cfg->exception_message = g_strdup (message);
280 ctx->cfg->disable_llvm = TRUE;
286 * The LLVM type with width == sizeof (gpointer)
291 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
297 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
303 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
309 * Return the size of the LLVM representation of the vtype T.
312 get_vtype_size (MonoType *t)
316 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
318 /* LLVMArgAsIArgs depends on this since it stores whole words */
319 while (size < 2 * sizeof (gpointer) && mono_is_power_of_two (size) == -1)
326 * simd_class_to_llvm_type:
328 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
331 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
333 if (!strcmp (klass->name, "Vector2d")) {
334 return LLVMVectorType (LLVMDoubleType (), 2);
335 } else if (!strcmp (klass->name, "Vector2l")) {
336 return LLVMVectorType (LLVMInt64Type (), 2);
337 } else if (!strcmp (klass->name, "Vector2ul")) {
338 return LLVMVectorType (LLVMInt64Type (), 2);
339 } else if (!strcmp (klass->name, "Vector4i")) {
340 return LLVMVectorType (LLVMInt32Type (), 4);
341 } else if (!strcmp (klass->name, "Vector4ui")) {
342 return LLVMVectorType (LLVMInt32Type (), 4);
343 } else if (!strcmp (klass->name, "Vector4f")) {
344 return LLVMVectorType (LLVMFloatType (), 4);
345 } else if (!strcmp (klass->name, "Vector8s")) {
346 return LLVMVectorType (LLVMInt16Type (), 8);
347 } else if (!strcmp (klass->name, "Vector8us")) {
348 return LLVMVectorType (LLVMInt16Type (), 8);
349 } else if (!strcmp (klass->name, "Vector16sb")) {
350 return LLVMVectorType (LLVMInt8Type (), 16);
351 } else if (!strcmp (klass->name, "Vector16b")) {
352 return LLVMVectorType (LLVMInt8Type (), 16);
354 printf ("%s\n", klass->name);
360 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
361 static inline G_GNUC_UNUSED LLVMTypeRef
362 type_to_simd_type (int type)
366 return LLVMVectorType (LLVMInt8Type (), 16);
368 return LLVMVectorType (LLVMInt16Type (), 8);
370 return LLVMVectorType (LLVMInt32Type (), 4);
372 return LLVMVectorType (LLVMInt64Type (), 2);
374 return LLVMVectorType (LLVMDoubleType (), 2);
376 return LLVMVectorType (LLVMFloatType (), 4);
378 g_assert_not_reached ();
384 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
386 int i, size, nfields, esize;
387 LLVMTypeRef *eltypes;
392 t = &klass->byval_arg;
394 if (mini_type_is_hfa (t, &nfields, &esize)) {
396 * This is needed on arm64 where HFAs are returned in
400 eltypes = g_new (LLVMTypeRef, size);
401 for (i = 0; i < size; ++i)
402 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
404 size = get_vtype_size (t);
406 eltypes = g_new (LLVMTypeRef, size);
407 for (i = 0; i < size; ++i)
408 eltypes [i] = LLVMInt8Type ();
411 name = mono_type_full_name (&klass->byval_arg);
412 ltype = LLVMStructCreateNamed (module->context, name);
413 LLVMStructSetBody (ltype, eltypes, size, FALSE);
423 * Return the LLVM type corresponding to T.
426 type_to_llvm_type (EmitContext *ctx, MonoType *t)
428 t = mini_get_underlying_type (t);
432 return LLVMVoidType ();
434 return LLVMInt8Type ();
436 return LLVMInt16Type ();
438 return LLVMInt32Type ();
440 return LLVMInt8Type ();
442 return LLVMInt16Type ();
444 return LLVMInt32Type ();
445 case MONO_TYPE_BOOLEAN:
446 return LLVMInt8Type ();
449 return LLVMInt64Type ();
451 return LLVMInt16Type ();
453 return LLVMFloatType ();
455 return LLVMDoubleType ();
458 return IntPtrType ();
459 case MONO_TYPE_OBJECT:
460 case MONO_TYPE_CLASS:
461 case MONO_TYPE_ARRAY:
462 case MONO_TYPE_SZARRAY:
463 case MONO_TYPE_STRING:
465 return ObjRefType ();
468 /* Because of generic sharing */
469 return ObjRefType ();
470 case MONO_TYPE_GENERICINST:
471 if (!mono_type_generic_inst_is_valuetype (t))
472 return ObjRefType ();
474 case MONO_TYPE_VALUETYPE:
475 case MONO_TYPE_TYPEDBYREF: {
479 klass = mono_class_from_mono_type (t);
481 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
482 return simd_class_to_llvm_type (ctx, klass);
485 return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
487 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
489 ltype = create_llvm_type_for_type (ctx->module, klass);
490 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
496 printf ("X: %d\n", t->type);
497 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
498 ctx->cfg->disable_llvm = TRUE;
506 * Return whenever T is an unsigned int type.
509 type_is_unsigned (EmitContext *ctx, MonoType *t)
511 t = mini_get_underlying_type (t);
527 * type_to_llvm_arg_type:
529 * Same as type_to_llvm_type, but treat i8/i16 as i32.
532 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
534 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
536 if (ctx->cfg->llvm_only)
540 * This works on all abis except arm64/ios which passes multiple
541 * arguments in one stack slot.
544 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
546 * LLVM generates code which only sets the lower bits, while JITted
547 * code expects all the bits to be set.
549 ptype = LLVMInt32Type ();
557 * llvm_type_to_stack_type:
559 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
562 static G_GNUC_UNUSED LLVMTypeRef
563 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
567 if (type == LLVMInt8Type ())
568 return LLVMInt32Type ();
569 else if (type == LLVMInt16Type ())
570 return LLVMInt32Type ();
571 else if (!cfg->r4fp && type == LLVMFloatType ())
572 return LLVMDoubleType ();
578 * regtype_to_llvm_type:
580 * Return the LLVM type corresponding to the regtype C used in instruction
584 regtype_to_llvm_type (char c)
588 return LLVMInt32Type ();
590 return LLVMInt64Type ();
592 return LLVMDoubleType ();
601 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
604 op_to_llvm_type (int opcode)
609 return LLVMInt8Type ();
612 return LLVMInt8Type ();
615 return LLVMInt16Type ();
618 return LLVMInt16Type ();
621 return LLVMInt32Type ();
624 return LLVMInt32Type ();
626 return LLVMInt64Type ();
628 return LLVMFloatType ();
630 return LLVMDoubleType ();
632 return LLVMInt64Type ();
634 return LLVMInt32Type ();
636 return LLVMInt64Type ();
641 return LLVMInt8Type ();
646 return LLVMInt16Type ();
648 return LLVMInt32Type ();
651 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
658 return LLVMInt32Type ();
665 return LLVMInt64Type ();
667 printf ("%s\n", mono_inst_name (opcode));
668 g_assert_not_reached ();
673 #define CLAUSE_START(clause) ((clause)->try_offset)
674 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
677 * load_store_to_llvm_type:
679 * Return the size/sign/zero extension corresponding to the load/store opcode
683 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
689 case OP_LOADI1_MEMBASE:
690 case OP_STOREI1_MEMBASE_REG:
691 case OP_STOREI1_MEMBASE_IMM:
692 case OP_ATOMIC_LOAD_I1:
693 case OP_ATOMIC_STORE_I1:
696 return LLVMInt8Type ();
697 case OP_LOADU1_MEMBASE:
699 case OP_ATOMIC_LOAD_U1:
700 case OP_ATOMIC_STORE_U1:
703 return LLVMInt8Type ();
704 case OP_LOADI2_MEMBASE:
705 case OP_STOREI2_MEMBASE_REG:
706 case OP_STOREI2_MEMBASE_IMM:
707 case OP_ATOMIC_LOAD_I2:
708 case OP_ATOMIC_STORE_I2:
711 return LLVMInt16Type ();
712 case OP_LOADU2_MEMBASE:
714 case OP_ATOMIC_LOAD_U2:
715 case OP_ATOMIC_STORE_U2:
718 return LLVMInt16Type ();
719 case OP_LOADI4_MEMBASE:
720 case OP_LOADU4_MEMBASE:
723 case OP_STOREI4_MEMBASE_REG:
724 case OP_STOREI4_MEMBASE_IMM:
725 case OP_ATOMIC_LOAD_I4:
726 case OP_ATOMIC_STORE_I4:
727 case OP_ATOMIC_LOAD_U4:
728 case OP_ATOMIC_STORE_U4:
730 return LLVMInt32Type ();
731 case OP_LOADI8_MEMBASE:
733 case OP_STOREI8_MEMBASE_REG:
734 case OP_STOREI8_MEMBASE_IMM:
735 case OP_ATOMIC_LOAD_I8:
736 case OP_ATOMIC_STORE_I8:
737 case OP_ATOMIC_LOAD_U8:
738 case OP_ATOMIC_STORE_U8:
740 return LLVMInt64Type ();
741 case OP_LOADR4_MEMBASE:
742 case OP_STORER4_MEMBASE_REG:
743 case OP_ATOMIC_LOAD_R4:
744 case OP_ATOMIC_STORE_R4:
746 return LLVMFloatType ();
747 case OP_LOADR8_MEMBASE:
748 case OP_STORER8_MEMBASE_REG:
749 case OP_ATOMIC_LOAD_R8:
750 case OP_ATOMIC_STORE_R8:
752 return LLVMDoubleType ();
753 case OP_LOAD_MEMBASE:
755 case OP_STORE_MEMBASE_REG:
756 case OP_STORE_MEMBASE_IMM:
757 *size = sizeof (gpointer);
758 return IntPtrType ();
760 g_assert_not_reached ();
768 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
771 ovf_op_to_intrins (int opcode)
775 return "llvm.sadd.with.overflow.i32";
777 return "llvm.uadd.with.overflow.i32";
779 return "llvm.ssub.with.overflow.i32";
781 return "llvm.usub.with.overflow.i32";
783 return "llvm.smul.with.overflow.i32";
785 return "llvm.umul.with.overflow.i32";
787 return "llvm.sadd.with.overflow.i64";
789 return "llvm.uadd.with.overflow.i64";
791 return "llvm.ssub.with.overflow.i64";
793 return "llvm.usub.with.overflow.i64";
795 return "llvm.smul.with.overflow.i64";
797 return "llvm.umul.with.overflow.i64";
799 g_assert_not_reached ();
805 simd_op_to_intrins (int opcode)
808 #if defined(TARGET_X86) || defined(TARGET_AMD64)
810 return "llvm.x86.sse2.min.pd";
812 return "llvm.x86.sse.min.ps";
814 return "llvm.x86.sse41.pminud";
816 return "llvm.x86.sse41.pminuw";
818 return "llvm.x86.sse2.pminu.b";
820 return "llvm.x86.sse2.pmins.w";
822 return "llvm.x86.sse2.max.pd";
824 return "llvm.x86.sse.max.ps";
826 return "llvm.x86.sse3.hadd.pd";
828 return "llvm.x86.sse3.hadd.ps";
830 return "llvm.x86.sse3.hsub.pd";
832 return "llvm.x86.sse3.hsub.ps";
834 return "llvm.x86.sse41.pmaxud";
836 return "llvm.x86.sse41.pmaxuw";
838 return "llvm.x86.sse2.pmaxu.b";
840 return "llvm.x86.sse3.addsub.ps";
842 return "llvm.x86.sse3.addsub.pd";
843 case OP_EXTRACT_MASK:
844 return "llvm.x86.sse2.pmovmskb.128";
847 return "llvm.x86.sse2.psrli.w";
850 return "llvm.x86.sse2.psrli.d";
853 return "llvm.x86.sse2.psrli.q";
856 return "llvm.x86.sse2.pslli.w";
859 return "llvm.x86.sse2.pslli.d";
862 return "llvm.x86.sse2.pslli.q";
865 return "llvm.x86.sse2.psrai.w";
868 return "llvm.x86.sse2.psrai.d";
870 return "llvm.x86.sse2.padds.b";
872 return "llvm.x86.sse2.padds.w";
874 return "llvm.x86.sse2.psubs.b";
876 return "llvm.x86.sse2.psubs.w";
877 case OP_PADDB_SAT_UN:
878 return "llvm.x86.sse2.paddus.b";
879 case OP_PADDW_SAT_UN:
880 return "llvm.x86.sse2.paddus.w";
881 case OP_PSUBB_SAT_UN:
882 return "llvm.x86.sse2.psubus.b";
883 case OP_PSUBW_SAT_UN:
884 return "llvm.x86.sse2.psubus.w";
886 return "llvm.x86.sse2.pavg.b";
888 return "llvm.x86.sse2.pavg.w";
890 return "llvm.x86.sse.sqrt.ps";
892 return "llvm.x86.sse2.sqrt.pd";
894 return "llvm.x86.sse.rsqrt.ps";
896 return "llvm.x86.sse.rcp.ps";
898 return "llvm.x86.sse2.cvtdq2pd";
900 return "llvm.x86.sse2.cvtdq2ps";
902 return "llvm.x86.sse2.cvtpd2dq";
904 return "llvm.x86.sse2.cvtps2dq";
906 return "llvm.x86.sse2.cvtpd2ps";
908 return "llvm.x86.sse2.cvtps2pd";
910 return "llvm.x86.sse2.cvttpd2dq";
912 return "llvm.x86.sse2.cvttps2dq";
914 return "llvm.x86.sse.cmp.ps";
916 return "llvm.x86.sse2.cmp.pd";
918 return "llvm.x86.sse2.packsswb.128";
920 return "llvm.x86.sse2.packssdw.128";
922 return "llvm.x86.sse2.packuswb.128";
924 return "llvm.x86.sse41.packusdw";
926 return "llvm.x86.sse2.pmulh.w";
927 case OP_PMULW_HIGH_UN:
928 return "llvm.x86.sse2.pmulhu.w";
931 g_assert_not_reached ();
937 simd_op_to_llvm_type (int opcode)
939 #if defined(TARGET_X86) || defined(TARGET_AMD64)
943 return type_to_simd_type (MONO_TYPE_R8);
946 return type_to_simd_type (MONO_TYPE_I8);
949 return type_to_simd_type (MONO_TYPE_I4);
954 return type_to_simd_type (MONO_TYPE_I2);
958 return type_to_simd_type (MONO_TYPE_I1);
960 return type_to_simd_type (MONO_TYPE_R4);
963 return type_to_simd_type (MONO_TYPE_I4);
967 return type_to_simd_type (MONO_TYPE_R8);
971 return type_to_simd_type (MONO_TYPE_R4);
972 case OP_EXTRACT_MASK:
973 return type_to_simd_type (MONO_TYPE_I1);
979 return type_to_simd_type (MONO_TYPE_R4);
982 return type_to_simd_type (MONO_TYPE_R8);
984 g_assert_not_reached ();
995 * Return the LLVM basic block corresponding to BB.
997 static LLVMBasicBlockRef
998 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
1000 char bb_name_buf [128];
1003 if (ctx->bblocks [bb->block_num].bblock == NULL) {
1004 if (bb->flags & BB_EXCEPTION_HANDLER) {
1005 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1006 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1007 bb_name = bb_name_buf;
1008 } else if (bb->block_num < 256) {
1009 if (!ctx->module->bb_names) {
1010 ctx->module->bb_names_len = 256;
1011 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1013 if (!ctx->module->bb_names [bb->block_num]) {
1016 n = g_strdup_printf ("BB%d", bb->block_num);
1017 mono_memory_barrier ();
1018 ctx->module->bb_names [bb->block_num] = n;
1020 bb_name = ctx->module->bb_names [bb->block_num];
1022 sprintf (bb_name_buf, "BB%d", bb->block_num);
1023 bb_name = bb_name_buf;
1026 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1027 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1030 return ctx->bblocks [bb->block_num].bblock;
1036 * Return the last LLVM bblock corresponding to BB.
1037 * This might not be equal to the bb returned by get_bb () since we need to generate
1038 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1040 static LLVMBasicBlockRef
1041 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1044 return ctx->bblocks [bb->block_num].end_bblock;
1047 static LLVMBasicBlockRef
1048 gen_bb (EmitContext *ctx, const char *prefix)
1052 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1053 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1059 * Return the target of the patch identified by TYPE and TARGET.
1062 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1068 memset (&ji, 0, sizeof (ji));
1070 ji.data.target = target;
1072 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, &error);
1073 mono_error_assert_ok (&error);
1081 * Emit code to convert the LLVM value V to DTYPE.
1084 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1086 LLVMTypeRef stype = LLVMTypeOf (v);
1088 if (stype != dtype) {
1089 gboolean ext = FALSE;
1092 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1094 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1096 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1100 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1102 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1103 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1106 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1107 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1108 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1109 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1110 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1111 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1112 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1113 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1115 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1116 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1117 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1118 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1119 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1120 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1122 if (mono_arch_is_soft_float ()) {
1123 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1124 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1125 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1126 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1129 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1130 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1133 LLVMDumpValue (LLVMConstNull (dtype));
1134 g_assert_not_reached ();
1142 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1144 return convert_full (ctx, v, dtype, FALSE);
1148 * emit_volatile_load:
1150 * If vreg is volatile, emit a load from its address.
1153 emit_volatile_load (EmitContext *ctx, int vreg)
1157 LLVMValueRef v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1158 t = ctx->vreg_cli_types [vreg];
1159 if (t && !t->byref) {
1161 * Might have to zero extend since llvm doesn't have
1164 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1165 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1166 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1167 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1168 else if (t->type == MONO_TYPE_U8)
1169 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1176 * emit_volatile_store:
1178 * If VREG is volatile, emit a store from its value to its address.
1181 emit_volatile_store (EmitContext *ctx, int vreg)
1183 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1185 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1186 g_assert (ctx->addresses [vreg]);
1187 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1192 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1194 LLVMTypeRef ret_type;
1195 LLVMTypeRef *param_types = NULL;
1200 rtype = mini_get_underlying_type (sig->ret);
1201 ret_type = type_to_llvm_type (ctx, rtype);
1205 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1209 param_types [pindex ++] = ThisType ();
1210 for (i = 0; i < sig->param_count; ++i)
1211 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1213 if (!ctx_ok (ctx)) {
1214 g_free (param_types);
1218 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1219 g_free (param_types);
1225 * sig_to_llvm_sig_full:
1227 * Return the LLVM signature corresponding to the mono signature SIG using the
1228 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1231 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1233 LLVMTypeRef ret_type;
1234 LLVMTypeRef *param_types = NULL;
1236 int i, j, pindex, vret_arg_pindex = 0;
1237 gboolean vretaddr = FALSE;
1241 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1243 rtype = mini_get_underlying_type (sig->ret);
1244 ret_type = type_to_llvm_type (ctx, rtype);
1248 switch (cinfo->ret.storage) {
1249 case LLVMArgVtypeInReg:
1250 /* LLVM models this by returning an aggregate value */
1251 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1252 LLVMTypeRef members [2];
1254 members [0] = IntPtrType ();
1255 ret_type = LLVMStructType (members, 1, FALSE);
1256 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1258 ret_type = LLVMVoidType ();
1259 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1260 LLVMTypeRef members [2];
1262 members [0] = IntPtrType ();
1263 members [1] = IntPtrType ();
1264 ret_type = LLVMStructType (members, 2, FALSE);
1266 g_assert_not_reached ();
1269 case LLVMArgVtypeByVal:
1270 /* Vtype returned normally by val */
1272 case LLVMArgVtypeAsScalar: {
1273 int size = mono_class_value_size (mono_class_from_mono_type (rtype), NULL);
1274 /* LLVM models this by returning an int */
1275 if (size < SIZEOF_VOID_P) {
1276 g_assert (cinfo->ret.nslots == 1);
1277 ret_type = LLVMIntType (size * 8);
1279 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1280 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (mgreg_t) * 8);
1284 case LLVMArgFpStruct: {
1285 /* Vtype returned as a fp struct */
1286 LLVMTypeRef members [16];
1288 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1289 for (i = 0; i < cinfo->ret.nslots; ++i)
1290 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1291 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1294 case LLVMArgVtypeByRef:
1295 /* Vtype returned using a hidden argument */
1296 ret_type = LLVMVoidType ();
1298 case LLVMArgVtypeRetAddr:
1299 case LLVMArgGsharedvtFixed:
1300 case LLVMArgGsharedvtFixedVtype:
1301 case LLVMArgGsharedvtVariable:
1303 ret_type = LLVMVoidType ();
1309 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1311 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1313 * Has to be the first argument because of the sret argument attribute
1314 * FIXME: This might conflict with passing 'this' as the first argument, but
1315 * this is only used on arm64 which has a dedicated struct return register.
1317 cinfo->vret_arg_pindex = pindex;
1318 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1319 if (!ctx_ok (ctx)) {
1320 g_free (param_types);
1323 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1326 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1327 cinfo->rgctx_arg_pindex = pindex;
1328 param_types [pindex] = ctx->module->ptr_type;
1331 if (cinfo->imt_arg) {
1332 cinfo->imt_arg_pindex = pindex;
1333 param_types [pindex] = ctx->module->ptr_type;
1337 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1338 vret_arg_pindex = pindex;
1339 if (cinfo->vret_arg_index == 1) {
1340 /* Add the slots consumed by the first argument */
1341 LLVMArgInfo *ainfo = &cinfo->args [0];
1342 switch (ainfo->storage) {
1343 case LLVMArgVtypeInReg:
1344 for (j = 0; j < 2; ++j) {
1345 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1354 cinfo->vret_arg_pindex = vret_arg_pindex;
1357 if (vretaddr && vret_arg_pindex == pindex)
1358 param_types [pindex ++] = IntPtrType ();
1360 cinfo->this_arg_pindex = pindex;
1361 param_types [pindex ++] = ThisType ();
1362 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1364 if (vretaddr && vret_arg_pindex == pindex)
1365 param_types [pindex ++] = IntPtrType ();
1366 for (i = 0; i < sig->param_count; ++i) {
1367 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1369 if (vretaddr && vret_arg_pindex == pindex)
1370 param_types [pindex ++] = IntPtrType ();
1371 ainfo->pindex = pindex;
1373 switch (ainfo->storage) {
1374 case LLVMArgVtypeInReg:
1375 for (j = 0; j < 2; ++j) {
1376 switch (ainfo->pair_storage [j]) {
1378 param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
1383 g_assert_not_reached ();
1387 case LLVMArgVtypeByVal:
1388 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1391 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1394 case LLVMArgAsIArgs:
1395 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1398 case LLVMArgVtypeByRef:
1399 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1402 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1405 case LLVMArgAsFpArgs: {
1408 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1409 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1410 param_types [pindex ++] = LLVMDoubleType ();
1411 for (j = 0; j < ainfo->nslots; ++j)
1412 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1415 case LLVMArgVtypeAsScalar:
1416 g_assert_not_reached ();
1418 case LLVMArgGsharedvtFixed:
1419 case LLVMArgGsharedvtFixedVtype:
1420 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1422 case LLVMArgGsharedvtVariable:
1423 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1426 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1430 if (!ctx_ok (ctx)) {
1431 g_free (param_types);
1434 if (vretaddr && vret_arg_pindex == pindex)
1435 param_types [pindex ++] = IntPtrType ();
1436 if (ctx->llvm_only && cinfo->rgctx_arg) {
1437 /* Pass the rgctx as the last argument */
1438 cinfo->rgctx_arg_pindex = pindex;
1439 param_types [pindex] = ctx->module->ptr_type;
1443 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1444 g_free (param_types);
1450 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1452 return sig_to_llvm_sig_full (ctx, sig, NULL);
1456 * LLVMFunctionType1:
1458 * Create an LLVM function type from the arguments.
1460 static G_GNUC_UNUSED LLVMTypeRef
1461 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1464 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1468 * LLVMFunctionType1:
1470 * Create an LLVM function type from the arguments.
1472 static G_GNUC_UNUSED LLVMTypeRef
1473 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1474 LLVMTypeRef ParamType1,
1477 LLVMTypeRef param_types [1];
1479 param_types [0] = ParamType1;
1481 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1485 * LLVMFunctionType2:
1487 * Create an LLVM function type from the arguments.
1489 static G_GNUC_UNUSED LLVMTypeRef
1490 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1491 LLVMTypeRef ParamType1,
1492 LLVMTypeRef ParamType2,
1495 LLVMTypeRef param_types [2];
1497 param_types [0] = ParamType1;
1498 param_types [1] = ParamType2;
1500 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1504 * LLVMFunctionType3:
1506 * Create an LLVM function type from the arguments.
1508 static G_GNUC_UNUSED LLVMTypeRef
1509 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1510 LLVMTypeRef ParamType1,
1511 LLVMTypeRef ParamType2,
1512 LLVMTypeRef ParamType3,
1515 LLVMTypeRef param_types [3];
1517 param_types [0] = ParamType1;
1518 param_types [1] = ParamType2;
1519 param_types [2] = ParamType3;
1521 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1524 static G_GNUC_UNUSED LLVMTypeRef
1525 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1526 LLVMTypeRef ParamType1,
1527 LLVMTypeRef ParamType2,
1528 LLVMTypeRef ParamType3,
1529 LLVMTypeRef ParamType4,
1530 LLVMTypeRef ParamType5,
1533 LLVMTypeRef param_types [5];
1535 param_types [0] = ParamType1;
1536 param_types [1] = ParamType2;
1537 param_types [2] = ParamType3;
1538 param_types [3] = ParamType4;
1539 param_types [4] = ParamType5;
1541 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1547 * Create an LLVM builder and remember it so it can be freed later.
1549 static LLVMBuilderRef
1550 create_builder (EmitContext *ctx)
1552 LLVMBuilderRef builder = LLVMCreateBuilder ();
1554 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1560 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1565 case MONO_PATCH_INFO_INTERNAL_METHOD:
1566 name = g_strdup_printf ("jit_icall_%s", data);
1568 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1569 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1570 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1574 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1582 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1586 LLVMValueRef indexes [2];
1588 LLVMValueRef got_entry_addr, load;
1589 LLVMBuilderRef builder = ctx->builder;
1594 ji = g_new0 (MonoJumpInfo, 1);
1596 ji->data.target = data;
1598 ji = mono_aot_patch_info_dup (ji);
1600 ji->next = cfg->patch_info;
1601 cfg->patch_info = ji;
1603 got_offset = mono_aot_get_got_offset (cfg->patch_info);
1604 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1606 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1607 * explicitly initialize it.
1609 if (!mono_aot_is_shared_got_offset (got_offset)) {
1610 //mono_print_ji (ji);
1612 ctx->has_got_access = TRUE;
1615 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1616 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1617 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1619 name = get_aotconst_name (type, data, got_offset);
1621 load = LLVMBuildLoad (builder, got_entry_addr, "");
1622 load = convert (ctx, load, llvm_type);
1623 LLVMSetValueName (load, name ? name : "");
1625 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1628 //set_invariant_load_flag (load);
1634 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1636 return get_aotconst_typed (ctx, type, data, NULL);
1640 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1642 LLVMValueRef callee;
1644 if (ctx->llvm_only) {
1645 callee_name = mono_aot_get_direct_call_symbol (type, data);
1647 /* Directly callable */
1649 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1651 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1653 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1655 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1657 /* LLVMTypeRef's are uniqued */
1658 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1659 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1661 g_free (callee_name);
1667 * Calls are made through the GOT.
1669 return get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
1671 MonoJumpInfo *ji = NULL;
1673 callee_name = mono_aot_get_plt_symbol (type, data);
1677 if (ctx->cfg->compile_aot)
1678 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1679 mono_add_patch_info (ctx->cfg, 0, type, data);
1682 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
1684 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1686 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1688 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
1691 if (ctx->cfg->compile_aot) {
1692 ji = g_new0 (MonoJumpInfo, 1);
1694 ji->data.target = data;
1696 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
1704 emit_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, gpointer target)
1706 #if LLVM_API_VERSION > 100
1707 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
1708 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
1709 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
1710 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
1713 LLVMValueRef callee = LLVMAddFunction (ctx->lmodule, "", llvm_sig);
1714 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
1720 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
1722 MonoMethodHeader *header = cfg->header;
1723 MonoExceptionClause *clause;
1727 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
1728 return (bb->region >> 8) - 1;
1731 for (i = 0; i < header->num_clauses; ++i) {
1732 clause = &header->clauses [i];
1734 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
1741 static MonoExceptionClause *
1742 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
1744 // Since they're sorted by nesting we just need
1745 // the first one that the bb is a member of
1746 MonoExceptionClause *last = NULL;
1748 for (int i = 0; i < cfg->header->num_clauses; i++) {
1749 MonoExceptionClause *curr = &cfg->header->clauses [i];
1751 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
1754 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset)) {
1755 if (last && CLAUSE_END(last) > CLAUSE_END(curr))
1769 set_metadata_flag (LLVMValueRef v, const char *flag_name)
1771 LLVMValueRef md_arg;
1774 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1775 md_arg = LLVMMDString ("mono", 4);
1776 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1780 set_invariant_load_flag (LLVMValueRef v)
1782 LLVMValueRef md_arg;
1784 const char *flag_name;
1786 // FIXME: Cache this
1787 flag_name = "invariant.load";
1788 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1789 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
1790 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1796 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
1800 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
1802 MonoCompile *cfg = ctx->cfg;
1803 LLVMValueRef lcall = NULL;
1804 LLVMBuilderRef builder = *builder_ref;
1805 MonoExceptionClause *clause;
1807 if (ctx->llvm_only) {
1808 clause = get_most_deep_clause (cfg, ctx, bb);
1811 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1814 * Have to use an invoke instead of a call, branching to the
1815 * handler bblock of the clause containing this bblock.
1817 intptr_t key = CLAUSE_END(clause);
1819 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
1821 // FIXME: Find the one that has the lowest end bound for the right start address
1822 // FIXME: Finally + nesting
1825 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
1828 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
1830 builder = ctx->builder = create_builder (ctx);
1831 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1833 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1837 int clause_index = get_handler_clause (cfg, bb);
1839 if (clause_index != -1) {
1840 MonoMethodHeader *header = cfg->header;
1841 MonoExceptionClause *ec = &header->clauses [clause_index];
1842 MonoBasicBlock *tblock;
1843 LLVMBasicBlockRef ex_bb, noex_bb;
1846 * Have to use an invoke instead of a call, branching to the
1847 * handler bblock of the clause containing this bblock.
1850 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1852 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
1855 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
1857 ex_bb = get_bb (ctx, tblock);
1859 noex_bb = gen_bb (ctx, "NOEX_BB");
1862 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1864 builder = ctx->builder = create_builder (ctx);
1865 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1867 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1872 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1873 ctx->builder = builder;
1877 *builder_ref = ctx->builder;
1883 emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting, BarrierKind barrier)
1885 const char *intrins_name;
1886 LLVMValueRef args [16], res;
1887 LLVMTypeRef addr_type;
1888 gboolean use_intrinsics = TRUE;
1890 #if LLVM_API_VERSION > 100
1891 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1892 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
1893 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, addr, LLVMConstNull (LLVMTypeOf (addr)), "");
1894 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
1895 *builder_ref = ctx->builder;
1896 use_intrinsics = FALSE;
1900 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
1901 LLVMAtomicOrdering ordering;
1904 case LLVM_BARRIER_NONE:
1905 ordering = LLVMAtomicOrderingNotAtomic;
1907 case LLVM_BARRIER_ACQ:
1908 ordering = LLVMAtomicOrderingAcquire;
1910 case LLVM_BARRIER_SEQ:
1911 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1914 g_assert_not_reached ();
1919 * We handle loads which can fault by calling a mono specific intrinsic
1920 * using an invoke, so they are handled properly inside try blocks.
1921 * We can't use this outside clauses, since LLVM optimizes intrinsics which
1922 * are marked with IntrReadArgMem.
1926 intrins_name = "llvm.mono.load.i8.p0i8";
1929 intrins_name = "llvm.mono.load.i16.p0i16";
1932 intrins_name = "llvm.mono.load.i32.p0i32";
1935 intrins_name = "llvm.mono.load.i64.p0i64";
1938 g_assert_not_reached ();
1941 addr_type = LLVMTypeOf (addr);
1942 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
1943 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1946 args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1947 args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1948 args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1949 res = emit_call (ctx, bb, builder_ref, get_intrinsic (ctx, intrins_name), args, 4);
1951 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
1952 res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
1953 else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
1954 res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
1961 * We emit volatile loads for loads which can fault, because otherwise
1962 * LLVM will generate invalid code when encountering a load from a
1965 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting, barrier);
1967 /* Mark it with a custom metadata */
1970 set_metadata_flag (res, "mono.faulting.load");
1978 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
1980 return emit_load_general (ctx, bb, builder_ref, size, addr, name, is_faulting, LLVM_BARRIER_NONE);
1984 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting, BarrierKind barrier)
1986 const char *intrins_name;
1987 LLVMValueRef args [16];
1988 gboolean use_intrinsics = TRUE;
1990 #if LLVM_API_VERSION > 100
1991 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1992 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
1993 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, addr, LLVMConstNull (LLVMTypeOf (addr)), "");
1994 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
1995 *builder_ref = ctx->builder;
1996 use_intrinsics = FALSE;
2000 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
2001 LLVMAtomicOrdering ordering;
2004 case LLVM_BARRIER_NONE:
2005 ordering = LLVMAtomicOrderingNotAtomic;
2007 case LLVM_BARRIER_REL:
2008 ordering = LLVMAtomicOrderingRelease;
2010 case LLVM_BARRIER_SEQ:
2011 ordering = LLVMAtomicOrderingSequentiallyConsistent;
2014 g_assert_not_reached ();
2020 intrins_name = "llvm.mono.store.i8.p0i8";
2023 intrins_name = "llvm.mono.store.i16.p0i16";
2026 intrins_name = "llvm.mono.store.i32.p0i32";
2029 intrins_name = "llvm.mono.store.i64.p0i64";
2032 g_assert_not_reached ();
2035 if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
2036 value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
2037 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
2042 args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2043 args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
2044 args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
2045 emit_call (ctx, bb, builder_ref, get_intrinsic (ctx, intrins_name), args, 5);
2047 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
2052 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting)
2054 emit_store_general (ctx, bb, builder_ref, size, value, addr, is_faulting, LLVM_BARRIER_NONE);
2058 * emit_cond_system_exception:
2060 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2061 * Might set the ctx exception.
2064 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
2066 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2067 LLVMBuilderRef builder;
2068 MonoClass *exc_class;
2069 LLVMValueRef args [2];
2070 LLVMValueRef callee;
2071 gboolean no_pc = FALSE;
2073 if (IS_TARGET_AMD64)
2074 /* Some platforms don't require the pc argument */
2077 ex_bb = gen_bb (ctx, "EX_BB");
2079 ex2_bb = gen_bb (ctx, "EX2_BB");
2080 noex_bb = gen_bb (ctx, "NOEX_BB");
2082 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2084 exc_class = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2086 /* Emit exception throwing code */
2087 ctx->builder = builder = create_builder (ctx);
2088 LLVMPositionBuilderAtEnd (builder, ex_bb);
2090 if (ctx->cfg->llvm_only) {
2091 static LLVMTypeRef sig;
2094 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2095 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, "mono_llvm_throw_corlib_exception");
2097 LLVMBuildBr (builder, ex2_bb);
2099 ctx->builder = builder = create_builder (ctx);
2100 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2102 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2103 emit_call (ctx, bb, &builder, callee, args, 1);
2104 LLVMBuildUnreachable (builder);
2106 ctx->builder = builder = create_builder (ctx);
2107 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2109 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2115 callee = ctx->module->throw_corlib_exception;
2118 const char *icall_name;
2121 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2123 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2124 icall_name = "llvm_throw_corlib_exception_abs_trampoline";
2126 if (ctx->cfg->compile_aot) {
2127 callee = get_callee (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
2130 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2131 * - On x86, LLVM generated code doesn't push the arguments
2132 * - The trampoline takes the throw address as an arguments, not a pc offset.
2134 gpointer target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
2135 callee = emit_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, target);
2137 #if LLVM_API_VERSION > 100
2139 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2140 * added by emit_jit_callee ().
2142 ex2_bb = gen_bb (ctx, "EX2_BB");
2143 LLVMBuildBr (builder, ex2_bb);
2146 ctx->builder = builder = create_builder (ctx);
2147 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2149 mono_memory_barrier ();
2150 ctx->module->throw_corlib_exception = callee;
2155 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2158 * The LLVM mono branch contains changes so a block address can be passed as an
2159 * argument to a call.
2162 emit_call (ctx, bb, &builder, callee, args, 1);
2164 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2165 emit_call (ctx, bb, &builder, callee, args, 2);
2168 LLVMBuildUnreachable (builder);
2170 ctx->builder = builder = create_builder (ctx);
2171 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2173 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2180 * emit_args_to_vtype:
2182 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2185 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2187 int j, size, nslots;
2189 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
2191 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2192 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2195 if (ainfo->storage == LLVMArgAsFpArgs)
2196 nslots = ainfo->nslots;
2200 for (j = 0; j < nslots; ++j) {
2201 LLVMValueRef index [2], addr, daddr;
2202 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2203 LLVMTypeRef part_type;
2205 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2208 if (ainfo->pair_storage [j] == LLVMArgNone)
2211 switch (ainfo->pair_storage [j]) {
2212 case LLVMArgInIReg: {
2213 part_type = LLVMIntType (part_size * 8);
2214 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2215 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2216 addr = LLVMBuildGEP (builder, address, index, 1, "");
2218 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2219 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2220 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2222 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2225 case LLVMArgInFPReg: {
2226 LLVMTypeRef arg_type;
2228 if (ainfo->esize == 8)
2229 arg_type = LLVMDoubleType ();
2231 arg_type = LLVMFloatType ();
2233 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2234 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2235 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2236 LLVMBuildStore (builder, args [j], addr);
2242 g_assert_not_reached ();
2245 size -= sizeof (gpointer);
2250 * emit_vtype_to_args:
2252 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2253 * into ARGS, and the number of arguments into NARGS.
2256 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2259 int j, size, nslots;
2260 LLVMTypeRef arg_type;
2262 size = get_vtype_size (t);
2264 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t)))
2265 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2267 if (ainfo->storage == LLVMArgAsFpArgs)
2268 nslots = ainfo->nslots;
2271 for (j = 0; j < nslots; ++j) {
2272 LLVMValueRef index [2], addr, daddr;
2273 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2275 if (ainfo->pair_storage [j] == LLVMArgNone)
2278 switch (ainfo->pair_storage [j]) {
2280 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2281 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2282 addr = LLVMBuildGEP (builder, address, index, 1, "");
2284 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2285 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2286 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2288 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2290 case LLVMArgInFPReg:
2291 if (ainfo->esize == 8)
2292 arg_type = LLVMDoubleType ();
2294 arg_type = LLVMFloatType ();
2295 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2296 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2297 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2298 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2303 g_assert_not_reached ();
2305 size -= sizeof (gpointer);
2312 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2315 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2316 * get executed every time control reaches them.
2318 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2320 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2321 return ctx->last_alloca;
2325 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2327 return build_alloca_llvm_type_name (ctx, t, align, "");
2331 build_alloca (EmitContext *ctx, MonoType *t)
2333 MonoClass *k = mono_class_from_mono_type (t);
2336 g_assert (!mini_is_gsharedvt_variable_type (t));
2338 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2341 align = mono_class_min_align (k);
2343 /* Sometimes align is not a power of 2 */
2344 while (mono_is_power_of_two (align) == -1)
2347 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2351 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2355 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2357 MonoCompile *cfg = ctx->cfg;
2358 LLVMBuilderRef builder = ctx->builder;
2359 LLVMValueRef offset, offset_var;
2360 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2361 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2365 g_assert (info_var);
2366 g_assert (locals_var);
2368 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2370 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * sizeof (gpointer)), FALSE);
2371 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2373 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2374 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2376 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2380 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2383 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2386 module->used = g_ptr_array_sized_new (16);
2387 g_ptr_array_add (module->used, global);
2391 emit_llvm_used (MonoLLVMModule *module)
2393 LLVMModuleRef lmodule = module->lmodule;
2394 LLVMTypeRef used_type;
2395 LLVMValueRef used, *used_elem;
2401 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2402 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2403 used_elem = g_new0 (LLVMValueRef, module->used->len);
2404 for (i = 0; i < module->used->len; ++i)
2405 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2406 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2407 LLVMSetLinkage (used, LLVMAppendingLinkage);
2408 LLVMSetSection (used, "llvm.metadata");
2414 * Emit a function mapping method indexes to their code
2417 emit_get_method (MonoLLVMModule *module)
2419 LLVMModuleRef lmodule = module->lmodule;
2420 LLVMValueRef func, switch_ins, m;
2421 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2422 LLVMBasicBlockRef *bbs;
2424 LLVMBuilderRef builder;
2429 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2430 * but generating code seems safer.
2432 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2433 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2434 LLVMSetLinkage (func, LLVMExternalLinkage);
2435 LLVMSetVisibility (func, LLVMHiddenVisibility);
2436 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2437 module->get_method = func;
2439 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2442 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2443 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2444 * then we will have to find another solution.
2447 name = g_strdup_printf ("BB_CODE_START");
2448 code_start_bb = LLVMAppendBasicBlock (func, name);
2450 builder = LLVMCreateBuilder ();
2451 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2452 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2454 name = g_strdup_printf ("BB_CODE_END");
2455 code_end_bb = LLVMAppendBasicBlock (func, name);
2457 builder = LLVMCreateBuilder ();
2458 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2459 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2461 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2462 for (i = 0; i < module->max_method_idx + 1; ++i) {
2463 name = g_strdup_printf ("BB_%d", i);
2464 bb = LLVMAppendBasicBlock (func, name);
2468 builder = LLVMCreateBuilder ();
2469 LLVMPositionBuilderAtEnd (builder, bb);
2471 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2473 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2475 LLVMBuildRet (builder, LLVMConstNull (rtype));
2478 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2479 builder = LLVMCreateBuilder ();
2480 LLVMPositionBuilderAtEnd (builder, fail_bb);
2481 LLVMBuildRet (builder, LLVMConstNull (rtype));
2483 builder = LLVMCreateBuilder ();
2484 LLVMPositionBuilderAtEnd (builder, entry_bb);
2486 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2487 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2488 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2489 for (i = 0; i < module->max_method_idx + 1; ++i) {
2490 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2493 mark_as_used (module, func);
2497 * emit_get_unbox_tramp:
2499 * Emit a function mapping method indexes to their unbox trampoline
2502 emit_get_unbox_tramp (MonoLLVMModule *module)
2504 LLVMModuleRef lmodule = module->lmodule;
2505 LLVMValueRef func, switch_ins, m;
2506 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2507 LLVMBasicBlockRef *bbs;
2509 LLVMBuilderRef builder;
2513 /* Similar to emit_get_method () */
2515 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2516 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2517 LLVMSetLinkage (func, LLVMExternalLinkage);
2518 LLVMSetVisibility (func, LLVMHiddenVisibility);
2519 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2520 module->get_unbox_tramp = func;
2522 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2524 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2525 for (i = 0; i < module->max_method_idx + 1; ++i) {
2526 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2530 name = g_strdup_printf ("BB_%d", i);
2531 bb = LLVMAppendBasicBlock (func, name);
2535 builder = LLVMCreateBuilder ();
2536 LLVMPositionBuilderAtEnd (builder, bb);
2538 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2541 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2542 builder = LLVMCreateBuilder ();
2543 LLVMPositionBuilderAtEnd (builder, fail_bb);
2544 LLVMBuildRet (builder, LLVMConstNull (rtype));
2546 builder = LLVMCreateBuilder ();
2547 LLVMPositionBuilderAtEnd (builder, entry_bb);
2549 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2550 for (i = 0; i < module->max_method_idx + 1; ++i) {
2551 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2555 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2558 mark_as_used (module, func);
2561 /* Add a function to mark the beginning of LLVM code */
2563 emit_llvm_code_start (MonoLLVMModule *module)
2565 LLVMModuleRef lmodule = module->lmodule;
2567 LLVMBasicBlockRef entry_bb;
2568 LLVMBuilderRef builder;
2570 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2571 LLVMSetLinkage (func, LLVMInternalLinkage);
2572 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2573 module->code_start = func;
2574 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2575 builder = LLVMCreateBuilder ();
2576 LLVMPositionBuilderAtEnd (builder, entry_bb);
2577 LLVMBuildRetVoid (builder);
2581 emit_init_icall_wrapper (MonoLLVMModule *module, const char *name, const char *icall_name, int subtype)
2583 LLVMModuleRef lmodule = module->lmodule;
2584 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2585 LLVMBasicBlockRef entry_bb;
2586 LLVMBuilderRef builder;
2593 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
2594 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
2599 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
2600 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
2603 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
2604 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
2607 g_assert_not_reached ();
2609 LLVMSetLinkage (func, LLVMInternalLinkage);
2610 LLVMAddFunctionAttr (func, LLVMNoInlineAttribute);
2611 mono_llvm_set_preserveall_cc (func);
2612 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2613 builder = LLVMCreateBuilder ();
2614 LLVMPositionBuilderAtEnd (builder, entry_bb);
2617 ji = g_new0 (MonoJumpInfo, 1);
2618 ji->type = MONO_PATCH_INFO_AOT_MODULE;
2619 ji = mono_aot_patch_info_dup (ji);
2620 got_offset = mono_aot_get_got_offset (ji);
2621 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2622 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2623 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2624 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2625 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
2626 args [1] = LLVMGetParam (func, 0);
2628 args [2] = LLVMGetParam (func, 1);
2630 ji = g_new0 (MonoJumpInfo, 1);
2631 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
2632 ji->data.name = icall_name;
2633 ji = mono_aot_patch_info_dup (ji);
2634 got_offset = mono_aot_get_got_offset (ji);
2635 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2636 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2637 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2638 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2639 callee = LLVMBuildLoad (builder, got_entry_addr, "");
2640 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
2641 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
2643 // Set the inited flag
2644 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2645 indexes [1] = LLVMGetParam (func, 0);
2646 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
2648 LLVMBuildRetVoid (builder);
2650 LLVMVerifyFunction(func, LLVMAbortProcessAction);
2655 * Emit wrappers around the C icalls used to initialize llvm methods, to
2656 * make the calling code smaller and to enable usage of the llvm
2657 * PreserveAll calling convention.
2660 emit_init_icall_wrappers (MonoLLVMModule *module)
2662 module->init_method = emit_init_icall_wrapper (module, "init_method", "mono_aot_init_llvm_method", 0);
2663 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, "init_method_gshared_mrgctx", "mono_aot_init_gshared_method_mrgctx", 1);
2664 module->init_method_gshared_this = emit_init_icall_wrapper (module, "init_method_gshared_this", "mono_aot_init_gshared_method_this", 2);
2665 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, "init_method_gshared_vtable", "mono_aot_init_gshared_method_vtable", 3);
2669 emit_llvm_code_end (MonoLLVMModule *module)
2671 LLVMModuleRef lmodule = module->lmodule;
2673 LLVMBasicBlockRef entry_bb;
2674 LLVMBuilderRef builder;
2676 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2677 LLVMSetLinkage (func, LLVMInternalLinkage);
2678 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2679 module->code_end = func;
2680 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2681 builder = LLVMCreateBuilder ();
2682 LLVMPositionBuilderAtEnd (builder, entry_bb);
2683 LLVMBuildRetVoid (builder);
2687 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
2689 gboolean need_div_check = ctx->cfg->backend->need_div_check;
2692 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
2693 need_div_check = TRUE;
2695 if (!need_div_check)
2698 switch (ins->opcode) {
2711 case OP_IDIV_UN_IMM:
2712 case OP_LDIV_UN_IMM:
2713 case OP_IREM_UN_IMM:
2714 case OP_LREM_UN_IMM: {
2716 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
2717 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
2719 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
2720 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
2723 builder = ctx->builder;
2725 /* b == -1 && a == 0x80000000 */
2727 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
2728 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
2729 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
2731 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
2732 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
2735 builder = ctx->builder;
2747 * Emit code to initialize the GOT slots used by the method.
2750 emit_init_method (EmitContext *ctx)
2752 LLVMValueRef indexes [16], args [16], callee;
2753 LLVMValueRef inited_var, cmp, call;
2754 LLVMBasicBlockRef inited_bb, notinited_bb;
2755 LLVMBuilderRef builder = ctx->builder;
2756 MonoCompile *cfg = ctx->cfg;
2758 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
2760 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2761 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
2762 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
2764 args [0] = inited_var;
2765 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
2766 inited_var = LLVMBuildCall (ctx->builder, get_intrinsic (ctx, "llvm.expect.i8"), args, 2, "");
2768 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
2770 inited_bb = ctx->inited_bb;
2771 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
2773 LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
2775 builder = ctx->builder = create_builder (ctx);
2776 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
2779 if (ctx->rgctx_arg && cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) {
2780 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2781 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2782 callee = ctx->module->init_method_gshared_mrgctx;
2783 call = LLVMBuildCall (builder, callee, args, 2, "");
2784 } else if (ctx->rgctx_arg) {
2785 /* A vtable is passed as the rgctx argument */
2786 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2787 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2788 callee = ctx->module->init_method_gshared_vtable;
2789 call = LLVMBuildCall (builder, callee, args, 2, "");
2790 } else if (cfg->gshared) {
2791 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2792 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
2793 callee = ctx->module->init_method_gshared_this;
2794 call = LLVMBuildCall (builder, callee, args, 2, "");
2796 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2797 callee = ctx->module->init_method;
2798 call = LLVMBuildCall (builder, callee, args, 1, "");
2802 * This enables llvm to keep arguments in their original registers/
2803 * scratch registers, since the call will not clobber them.
2805 mono_llvm_set_call_preserveall_cc (call);
2807 LLVMBuildBr (builder, inited_bb);
2808 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
2810 builder = ctx->builder = create_builder (ctx);
2811 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
2815 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
2818 * Emit unbox trampoline using a tail call
2820 LLVMValueRef tramp, call, *args;
2821 LLVMBuilderRef builder;
2822 LLVMBasicBlockRef lbb;
2823 LLVMCallInfo *linfo;
2827 tramp_name = g_strdup_printf ("ut_%s", method_name);
2828 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
2829 LLVMSetLinkage (tramp, LLVMInternalLinkage);
2830 LLVMAddFunctionAttr (tramp, LLVMOptimizeForSizeAttribute);
2831 //LLVMAddFunctionAttr (tramp, LLVMNoUnwindAttribute);
2833 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
2834 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2835 LLVMAddAttribute (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVMInRegAttribute);
2836 if (ctx->cfg->vret_addr) {
2837 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
2838 if (linfo->ret.storage == LLVMArgVtypeByRef) {
2839 LLVMAddAttribute (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVMStructRetAttribute);
2840 LLVMAddAttribute (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
2844 lbb = LLVMAppendBasicBlock (tramp, "");
2845 builder = LLVMCreateBuilder ();
2846 LLVMPositionBuilderAtEnd (builder, lbb);
2848 nargs = LLVMCountParamTypes (method_type);
2849 args = g_new0 (LLVMValueRef, nargs);
2850 for (i = 0; i < nargs; ++i) {
2851 args [i] = LLVMGetParam (tramp, i);
2852 if (i == ctx->this_arg_pindex) {
2853 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
2855 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
2856 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), sizeof (MonoObject), FALSE), "");
2857 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
2860 call = LLVMBuildCall (builder, method, args, nargs, "");
2861 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2862 LLVMAddInstrAttribute (call, 1 + ctx->rgctx_arg_pindex, LLVMInRegAttribute);
2863 if (linfo->ret.storage == LLVMArgVtypeByRef)
2864 LLVMAddInstrAttribute (call, 1 + linfo->vret_arg_pindex, LLVMStructRetAttribute);
2866 // FIXME: This causes assertions in clang
2867 //mono_llvm_set_must_tail (call);
2868 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
2869 LLVMBuildRetVoid (builder);
2871 LLVMBuildRet (builder, call);
2873 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
2879 * Emit code to load/convert arguments.
2882 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
2885 MonoCompile *cfg = ctx->cfg;
2886 MonoMethodSignature *sig = ctx->sig;
2887 LLVMCallInfo *linfo = ctx->linfo;
2891 LLVMBuilderRef old_builder = ctx->builder;
2892 ctx->builder = builder;
2894 ctx->alloca_builder = create_builder (ctx);
2897 * Handle indirect/volatile variables by allocating memory for them
2898 * using 'alloca', and storing their address in a temporary.
2900 for (i = 0; i < cfg->num_varinfo; ++i) {
2901 MonoInst *var = cfg->varinfo [i];
2904 if (var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET) {
2905 } else if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
2906 vtype = type_to_llvm_type (ctx, var->inst_vtype);
2909 /* Could be already created by an OP_VPHI */
2910 if (!ctx->addresses [var->dreg]) {
2911 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
2912 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
2914 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
2918 names = g_new (char *, sig->param_count);
2919 mono_method_get_param_names (cfg->method, (const char **) names);
2921 for (i = 0; i < sig->param_count; ++i) {
2922 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
2923 int reg = cfg->args [i + sig->hasthis]->dreg;
2926 pindex = ainfo->pindex;
2928 switch (ainfo->storage) {
2929 case LLVMArgVtypeInReg:
2930 case LLVMArgAsFpArgs: {
2931 LLVMValueRef args [8];
2934 pindex += ainfo->ndummy_fpargs;
2936 /* The argument is received as a set of int/fp arguments, store them into the real argument */
2937 memset (args, 0, sizeof (args));
2938 if (ainfo->storage == LLVMArgVtypeInReg) {
2939 args [0] = LLVMGetParam (ctx->lmethod, pindex);
2940 if (ainfo->pair_storage [1] != LLVMArgNone)
2941 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
2943 g_assert (ainfo->nslots <= 8);
2944 for (j = 0; j < ainfo->nslots; ++j)
2945 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
2947 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2949 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
2951 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2952 /* Treat these as normal values */
2953 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2957 case LLVMArgVtypeByVal: {
2958 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2960 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2961 /* Treat these as normal values */
2962 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2966 case LLVMArgVtypeByRef: {
2967 /* The argument is passed by ref */
2968 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2971 case LLVMArgAsIArgs: {
2972 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2975 /* The argument is received as an array of ints, store it into the real argument */
2976 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2978 size = mono_class_value_size (mono_class_from_mono_type (ainfo->type), NULL);
2979 if (size < SIZEOF_VOID_P) {
2980 /* The upper bits of the registers might not be valid */
2981 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
2982 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
2983 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
2985 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
2989 case LLVMArgVtypeAsScalar:
2990 g_assert_not_reached ();
2992 case LLVMArgGsharedvtFixed: {
2993 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
2994 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2997 name = g_strdup_printf ("arg_%s", names [i]);
2999 name = g_strdup_printf ("arg_%d", i);
3001 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
3004 case LLVMArgGsharedvtFixedVtype: {
3005 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3008 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3010 name = g_strdup_printf ("vtype_arg_%d", i);
3012 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3013 g_assert (ctx->addresses [reg]);
3014 LLVMSetValueName (ctx->addresses [reg], name);
3015 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3018 case LLVMArgGsharedvtVariable:
3019 /* The IR treats these as variables with addresses */
3020 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3023 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
3030 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3032 emit_volatile_store (ctx, cfg->args [0]->dreg);
3033 for (i = 0; i < sig->param_count; ++i)
3034 if (!mini_type_is_vtype (sig->params [i]))
3035 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3037 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3038 LLVMValueRef this_alloc;
3041 * The exception handling code needs the location where the this argument was
3042 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3043 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3044 * location into the LSDA.
3046 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3047 /* This volatile store will keep the alloca alive */
3048 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3050 set_metadata_flag (this_alloc, "mono.this");
3053 if (cfg->rgctx_var) {
3054 LLVMValueRef rgctx_alloc, store;
3057 * We handle the rgctx arg similarly to the this pointer.
3059 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3060 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3061 /* This volatile store will keep the alloca alive */
3062 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3064 set_metadata_flag (rgctx_alloc, "mono.this");
3067 /* Initialize the method if needed */
3068 if (cfg->compile_aot && ctx->llvm_only) {
3069 /* Emit a location for the initialization code */
3070 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3071 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3073 LLVMBuildBr (ctx->builder, ctx->init_bb);
3074 builder = ctx->builder = create_builder (ctx);
3075 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3076 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3079 /* Compute nesting between clauses */
3080 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3081 for (i = 0; i < cfg->header->num_clauses; ++i) {
3082 for (j = 0; j < cfg->header->num_clauses; ++j) {
3083 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3084 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3086 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3087 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3092 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3093 * it needs to continue normally, or return back to the exception handling system.
3095 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3099 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3102 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3103 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3104 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3106 if (bb->in_scount == 0) {
3109 sprintf (name, "finally_ind_bb%d", bb->block_num);
3110 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3111 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3113 ctx->bblocks [bb->block_num].finally_ind = val;
3115 /* Create a variable to hold the exception var */
3117 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3121 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3122 * LLVM bblock containing a landing pad causes problems for the
3123 * LLVM optimizer passes.
3125 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3126 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3128 ctx->builder = old_builder;
3132 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3134 MonoCompile *cfg = ctx->cfg;
3135 LLVMValueRef *values = ctx->values;
3136 LLVMValueRef *addresses = ctx->addresses;
3137 MonoCallInst *call = (MonoCallInst*)ins;
3138 MonoMethodSignature *sig = call->signature;
3139 LLVMValueRef callee = NULL, lcall;
3141 LLVMCallInfo *cinfo;
3145 LLVMTypeRef llvm_sig;
3147 gboolean is_virtual, calli, preserveall;
3148 LLVMBuilderRef builder = *builder_ref;
3150 if (call->signature->call_convention != MONO_CALL_DEFAULT) {
3151 set_failure (ctx, "non-default callconv");
3155 cinfo = call->cinfo;
3157 if (call->rgctx_arg_reg)
3158 cinfo->rgctx_arg = TRUE;
3159 if (call->imt_arg_reg)
3160 cinfo->imt_arg = TRUE;
3162 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3164 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3168 is_virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE || ins->opcode == OP_RCALL_MEMBASE);
3169 calli = !call->fptr_is_patch && (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG || ins->opcode == OP_RCALL_REG);
3171 preserveall = FALSE;
3173 /* FIXME: Avoid creating duplicate methods */
3175 if (ins->flags & MONO_INST_HAS_METHOD) {
3179 if (cfg->compile_aot) {
3180 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3182 set_failure (ctx, "can't encode patch");
3185 if (cfg->llvm_only && call->method->klass->image->assembly == ctx->module->assembly) {
3187 * Collect instructions representing the callee into a hash so they can be replaced
3188 * by the llvm method for the callee if the callee turns out to be direct
3189 * callable. Currently this only requires it to not fail llvm compilation.
3191 GSList *l = (GSList*)g_hash_table_lookup (ctx->method_to_callers, call->method);
3192 l = g_slist_prepend (l, callee);
3193 g_hash_table_insert (ctx->method_to_callers, call->method, l);
3197 static int tramp_index;
3200 name = g_strdup_printf ("tramp_%d", tramp_index);
3203 #if LLVM_API_VERSION > 100
3205 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3206 * Make all calls through a global. The address of the global will be saved in
3207 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3210 LLVMValueRef tramp_var = g_hash_table_lookup (ctx->jit_callees, call->method);
3213 mono_create_jit_trampoline (mono_domain_get (),
3214 call->method, &error);
3215 if (!mono_error_ok (&error))
3216 mono_error_raise_exception (&error); /* FIXME: Don't raise here */
3217 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3218 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3219 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3220 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3222 callee = LLVMBuildLoad (builder, tramp_var, "");
3225 mono_create_jit_trampoline (mono_domain_get (),
3226 call->method, &error);
3227 if (!mono_error_ok (&error))
3228 mono_error_raise_exception (&error); /* FIXME: Don't raise here */
3230 callee = LLVMAddFunction (ctx->lmodule, name, llvm_sig);
3233 if (!mono_error_ok (&error))
3234 mono_error_raise_exception (&error); /* FIXME: Don't raise here */
3235 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3240 if (!cfg->llvm_only && call->method && strstr (call->method->klass->name, "AsyncVoidMethodBuilder")) {
3241 /* LLVM miscompiles async methods */
3242 set_failure (ctx, "#13734");
3247 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
3253 memset (&ji, 0, sizeof (ji));
3254 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
3255 ji.data.target = info->name;
3257 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
3259 if (cfg->compile_aot) {
3260 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
3262 set_failure (ctx, "can't encode patch");
3266 target = (gpointer)mono_icall_get_wrapper (info);
3267 callee = emit_jit_callee (ctx, "", llvm_sig, target);
3270 if (cfg->compile_aot) {
3272 if (cfg->abs_patches) {
3273 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3275 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3277 set_failure (ctx, "can't encode patch");
3283 set_failure (ctx, "aot");
3287 #if LLVM_API_VERSION > 100
3288 if (cfg->abs_patches) {
3289 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3293 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, &error);
3294 mono_error_assert_ok (&error);
3295 callee = emit_jit_callee (ctx, "", llvm_sig, target);
3297 g_assert_not_reached ();
3300 g_assert_not_reached ();
3303 callee = LLVMAddFunction (ctx->lmodule, "", llvm_sig);
3305 if (cfg->abs_patches) {
3306 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3311 * FIXME: Some trampolines might have
3312 * their own calling convention on some platforms.
3314 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, &error);
3315 mono_error_assert_ok (&error);
3316 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3320 LLVMAddGlobalMapping (ctx->module->ee, callee, (gpointer)call->fptr);
3327 int size = sizeof (gpointer);
3330 g_assert (ins->inst_offset % size == 0);
3331 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3333 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3335 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3337 if (ins->flags & MONO_INST_HAS_METHOD) {
3342 * Collect and convert arguments
3344 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg;
3345 len = sizeof (LLVMValueRef) * nargs;
3346 args = (LLVMValueRef*)alloca (len);
3347 memset (args, 0, len);
3348 l = call->out_ireg_args;
3350 if (call->rgctx_arg_reg) {
3351 g_assert (values [call->rgctx_arg_reg]);
3352 g_assert (cinfo->rgctx_arg_pindex < nargs);
3354 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3355 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3356 * it using a volatile load.
3359 if (!ctx->imt_rgctx_loc)
3360 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3361 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3362 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
3364 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3367 if (call->imt_arg_reg) {
3368 g_assert (!ctx->llvm_only);
3369 g_assert (values [call->imt_arg_reg]);
3370 g_assert (cinfo->imt_arg_pindex < nargs);
3372 if (!ctx->imt_rgctx_loc)
3373 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3374 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3375 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
3377 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3380 switch (cinfo->ret.storage) {
3381 case LLVMArgGsharedvtVariable: {
3382 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
3384 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
3385 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
3387 g_assert (addresses [call->inst.dreg]);
3388 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3394 if (!addresses [call->inst.dreg])
3395 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3396 g_assert (cinfo->vret_arg_pindex < nargs);
3397 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3398 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3400 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
3406 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
3407 * use the real callee for argument type conversion.
3409 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
3410 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
3411 LLVMGetParamTypes (callee_type, param_types);
3413 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3416 LLVMArgInfo *ainfo = &call->cinfo->args [i];
3418 pindex = ainfo->pindex;
3420 regpair = (guint32)(gssize)(l->data);
3421 reg = regpair & 0xffffff;
3422 args [pindex] = values [reg];
3423 switch (ainfo->storage) {
3424 case LLVMArgVtypeInReg:
3425 case LLVMArgAsFpArgs: {
3429 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
3430 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
3431 pindex += ainfo->ndummy_fpargs;
3433 g_assert (addresses [reg]);
3434 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
3438 // FIXME: Get rid of the VMOVE
3441 case LLVMArgVtypeByVal:
3442 g_assert (addresses [reg]);
3443 args [pindex] = addresses [reg];
3445 case LLVMArgVtypeByRef: {
3446 g_assert (addresses [reg]);
3447 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3450 case LLVMArgAsIArgs:
3451 g_assert (addresses [reg]);
3452 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
3454 case LLVMArgVtypeAsScalar:
3455 g_assert_not_reached ();
3457 case LLVMArgGsharedvtFixed:
3458 case LLVMArgGsharedvtFixedVtype:
3459 g_assert (addresses [reg]);
3460 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3462 case LLVMArgGsharedvtVariable:
3463 g_assert (addresses [reg]);
3464 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
3467 g_assert (args [pindex]);
3468 if (i == 0 && sig->hasthis)
3469 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
3471 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
3474 g_assert (pindex <= nargs);
3479 // FIXME: Align call sites
3485 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
3488 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
3490 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
3491 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
3493 /* The two can't be used together, so use only one LLVM calling conv to pass them */
3494 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
3495 if (!sig->pinvoke && !cfg->llvm_only)
3496 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
3498 mono_llvm_set_call_preserveall_cc (lcall);
3500 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3501 LLVMAddInstrAttribute (lcall, 1 + cinfo->vret_arg_pindex, LLVMStructRetAttribute);
3502 if (!ctx->llvm_only && call->rgctx_arg_reg)
3503 LLVMAddInstrAttribute (lcall, 1 + cinfo->rgctx_arg_pindex, LLVMInRegAttribute);
3504 if (call->imt_arg_reg)
3505 LLVMAddInstrAttribute (lcall, 1 + cinfo->imt_arg_pindex, LLVMInRegAttribute);
3507 /* Add byval attributes if needed */
3508 for (i = 0; i < sig->param_count; ++i) {
3509 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
3511 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
3512 LLVMAddInstrAttribute (lcall, 1 + ainfo->pindex, LLVMByValAttribute);
3516 * Convert the result
3518 switch (cinfo->ret.storage) {
3519 case LLVMArgVtypeInReg: {
3520 LLVMValueRef regs [2];
3522 if (LLVMTypeOf (lcall) == LLVMVoidType ())
3526 if (!addresses [ins->dreg])
3527 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
3529 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
3530 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
3531 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
3532 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
3535 case LLVMArgVtypeByVal:
3536 if (!addresses [call->inst.dreg])
3537 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3538 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
3540 case LLVMArgFpStruct:
3541 if (!addresses [call->inst.dreg])
3542 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3543 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3545 case LLVMArgVtypeAsScalar:
3546 if (!addresses [call->inst.dreg])
3547 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3548 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3550 case LLVMArgVtypeRetAddr:
3551 case LLVMArgVtypeByRef:
3552 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret))) {
3553 /* Some opcodes like STOREX_MEMBASE access these by value */
3554 g_assert (addresses [call->inst.dreg]);
3555 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3558 case LLVMArgGsharedvtVariable:
3560 case LLVMArgGsharedvtFixed:
3561 case LLVMArgGsharedvtFixedVtype:
3562 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3565 if (sig->ret->type != MONO_TYPE_VOID)
3566 /* If the method returns an unsigned value, need to zext it */
3567 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
3571 *builder_ref = ctx->builder;
3575 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3577 const char *icall_name = rethrow ? "mono_llvm_rethrow_exception" : "mono_llvm_throw_exception";
3578 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3580 LLVMTypeRef exc_type = type_to_llvm_type (ctx, &mono_get_exception_class ()->byval_arg);
3583 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
3585 if (ctx->cfg->compile_aot) {
3586 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, icall_name);
3588 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3589 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3590 mono_memory_barrier ();
3593 ctx->module->rethrow = callee;
3595 ctx->module->throw_icall = callee;
3599 LLVMValueRef args [2];
3601 args [0] = convert (ctx, exc, exc_type);
3602 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
3604 LLVMBuildUnreachable (ctx->builder);
3606 ctx->builder = create_builder (ctx);
3610 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3612 MonoMethodSignature *throw_sig;
3613 LLVMValueRef callee, arg;
3614 const char *icall_name;
3616 callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3617 icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
3620 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
3621 throw_sig->ret = &mono_get_void_class ()->byval_arg;
3622 throw_sig->params [0] = &mono_get_object_class ()->byval_arg;
3623 if (ctx->cfg->compile_aot) {
3624 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3629 * LLVM doesn't push the exception argument, so we need a different
3632 target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline");
3634 target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3636 callee = emit_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), target);
3639 mono_memory_barrier ();
3640 #if LLVM_API_VERSION < 100
3642 ctx->module->rethrow = callee;
3644 ctx->module->throw_icall = callee;
3647 arg = convert (ctx, exc, type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg));
3648 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
3652 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
3654 const char *icall_name = "mono_llvm_resume_exception";
3655 LLVMValueRef callee = ctx->module->resume_eh;
3657 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3660 if (ctx->cfg->compile_aot) {
3661 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3663 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3664 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3665 mono_memory_barrier ();
3667 ctx->module->resume_eh = callee;
3671 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
3673 LLVMBuildUnreachable (ctx->builder);
3675 ctx->builder = create_builder (ctx);
3679 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3681 const char *icall_name = "mono_llvm_clear_exception";
3683 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3684 LLVMValueRef callee = NULL;
3687 if (ctx->cfg->compile_aot) {
3688 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3690 // FIXME: This is broken.
3691 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3695 g_assert (builder && callee);
3697 return LLVMBuildCall (builder, callee, NULL, 0, "");
3701 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3703 const char *icall_name = "mono_llvm_load_exception";
3705 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
3706 LLVMValueRef callee = NULL;
3709 if (ctx->cfg->compile_aot) {
3710 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3712 // FIXME: This is broken.
3713 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3717 g_assert (builder && callee);
3719 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
3724 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
3726 const char *icall_name = "mono_llvm_match_exception";
3728 ctx->builder = builder;
3730 const int num_args = 5;
3731 LLVMValueRef args [num_args];
3732 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
3733 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
3734 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
3735 if (ctx->cfg->rgctx_var) {
3736 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
3737 g_assert (rgctx_alloc);
3738 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
3740 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
3743 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
3745 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
3747 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
3748 LLVMValueRef callee = ctx->module->match_exc;
3751 if (ctx->cfg->compile_aot) {
3752 ctx->builder = builder;
3753 // get_callee expects ctx->builder to be the emitting builder
3754 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3756 callee = ctx->module->match_exc = LLVMAddFunction (ctx->lmodule, icall_name, match_sig);
3757 LLVMAddGlobalMapping (ctx->module->ee, ctx->module->match_exc, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3758 ctx->module->match_exc = callee;
3759 mono_memory_barrier ();
3763 g_assert (builder && callee);
3765 g_assert (ctx->ex_var);
3767 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
3770 // FIXME: This won't work because the code-finding makes this
3772 /*#define MONO_PERSONALITY_DEBUG*/
3774 #ifdef MONO_PERSONALITY_DEBUG
3775 static const gboolean use_debug_personality = TRUE;
3776 static const char *default_personality_name = "mono_debug_personality";
3778 static const gboolean use_debug_personality = FALSE;
3779 static const char *default_personality_name = "__gxx_personality_v0";
3783 default_cpp_lpad_exc_signature (void)
3785 static gboolean inited = FALSE;
3786 static LLVMTypeRef sig;
3789 LLVMTypeRef signature [2];
3790 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
3791 signature [1] = LLVMInt32Type ();
3792 sig = LLVMStructType (signature, 2, FALSE);
3800 get_mono_personality (EmitContext *ctx)
3802 LLVMValueRef personality = NULL;
3803 static gint32 mapping_inited = FALSE;
3804 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
3806 if (!use_debug_personality) {
3807 if (ctx->cfg->compile_aot) {
3808 personality = get_intrinsic (ctx, default_personality_name);
3809 } else if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0) {
3810 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3811 LLVMAddGlobalMapping (ctx->module->ee, personality, personality);
3814 if (ctx->cfg->compile_aot) {
3815 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name);
3817 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3818 LLVMAddGlobalMapping (ctx->module->ee, personality, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name));
3819 mono_memory_barrier ();
3823 g_assert (personality);
3827 static LLVMBasicBlockRef
3828 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
3830 MonoCompile *cfg = ctx->cfg;
3831 LLVMBuilderRef old_builder = ctx->builder;
3832 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
3834 LLVMBuilderRef lpadBuilder = create_builder (ctx);
3835 ctx->builder = lpadBuilder;
3837 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
3838 g_assert (handler_bb);
3840 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3841 LLVMValueRef personality = get_mono_personality (ctx);
3842 g_assert (personality);
3844 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
3845 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
3847 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3848 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
3849 g_assert (landing_pad);
3851 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
3852 LLVMAddClause (landing_pad, cast);
3854 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
3855 LLVMBuilderRef resume_builder = create_builder (ctx);
3856 ctx->builder = resume_builder;
3857 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
3859 emit_resume_eh (ctx, handler_bb);
3862 ctx->builder = lpadBuilder;
3863 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3865 gboolean finally_only = TRUE;
3867 MonoExceptionClause *group_cursor = group_start;
3869 for (int i = 0; i < group_size; i ++) {
3870 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY))
3871 finally_only = FALSE;
3877 // Handle landing pad inlining
3879 if (!finally_only) {
3880 // So at each level of the exception stack we will match the exception again.
3881 // During that match, we need to compare against the handler types for the current
3882 // protected region. We send the try start and end so that we can only check against
3883 // handlers for this lexical protected region.
3884 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
3886 // if returns -1, resume
3887 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
3889 // else move to that target bb
3890 for (int i=0; i < group_size; i++) {
3891 MonoExceptionClause *clause = group_start + i;
3892 int clause_index = clause - cfg->header->clauses;
3893 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3894 g_assert (handler_bb);
3895 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3896 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3899 int clause_index = group_start - cfg->header->clauses;
3900 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3901 g_assert (finally_bb);
3903 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
3906 ctx->builder = old_builder;
3913 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
3915 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3916 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
3918 // Make exception available to catch blocks
3919 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY)) {
3920 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
3922 g_assert (ctx->ex_var);
3923 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
3925 if (bb->in_scount == 1) {
3926 MonoInst *exvar = bb->in_stack [0];
3927 g_assert (!ctx->values [exvar->dreg]);
3928 g_assert (ctx->ex_var);
3929 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
3930 emit_volatile_store (ctx, exvar->dreg);
3933 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
3936 LLVMBuilderRef handler_builder = create_builder (ctx);
3937 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
3938 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
3940 // Make the handler code end with a jump to cbb
3941 LLVMBuildBr (handler_builder, cbb);
3945 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
3947 MonoCompile *cfg = ctx->cfg;
3948 LLVMValueRef *values = ctx->values;
3949 LLVMModuleRef lmodule = ctx->lmodule;
3950 BBInfo *bblocks = ctx->bblocks;
3952 LLVMValueRef personality;
3953 LLVMValueRef landing_pad;
3954 LLVMBasicBlockRef target_bb;
3956 static int ti_generator;
3958 LLVMValueRef type_info;
3962 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3964 if (cfg->compile_aot) {
3965 /* Use a dummy personality function */
3966 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3967 g_assert (personality);
3969 #if LLVM_API_VERSION > 100
3970 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
3971 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
3972 LLVMAddFunctionAttr (personality, LLVMNoUnwindAttribute);
3973 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
3974 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
3975 LLVMPositionBuilderAtEnd (builder2, entry_bb);
3976 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
3978 static gint32 mapping_inited;
3980 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3982 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
3983 LLVMAddGlobalMapping (ctx->module->ee, personality, (gpointer)mono_personality);
3987 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
3989 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
3992 * Create the type info
3994 sprintf (ti_name, "type_info_%d", ti_generator);
3997 if (cfg->compile_aot) {
3998 /* decode_eh_frame () in aot-runtime.c will decode this */
3999 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4000 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4003 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4005 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4007 #if LLVM_API_VERSION > 100
4008 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4009 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4014 * After the cfg mempool is freed, the type info will point to stale memory,
4015 * but this is not a problem, since we decode it once in exception_cb during
4018 ti = (gint32*)mono_mempool_alloc (cfg->mempool, sizeof (gint32));
4019 *(gint32*)ti = clause_index;
4021 type_info = LLVMAddGlobal (lmodule, i8ptr, ti_name);
4023 LLVMAddGlobalMapping (ctx->module->ee, type_info, ti);
4028 LLVMTypeRef members [2], ret_type;
4030 members [0] = i8ptr;
4031 members [1] = LLVMInt32Type ();
4032 ret_type = LLVMStructType (members, 2, FALSE);
4034 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4035 LLVMAddClause (landing_pad, type_info);
4037 /* Store the exception into the exvar */
4039 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4043 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4044 * code expects control to be transferred to this landing pad even in the
4045 * presence of nested clauses. The landing pad needs to branch to the landing
4046 * pads belonging to nested clauses based on the selector value returned by
4047 * the landing pad instruction, which is passed to the landing pad in a
4048 * register by the EH code.
4050 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4051 g_assert (target_bb);
4054 * Branch to the correct landing pad
4056 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4057 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4059 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4060 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4061 MonoBasicBlock *handler_bb;
4063 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4064 g_assert (handler_bb);
4066 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4067 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4070 /* Start a new bblock which CALL_HANDLER can branch to */
4071 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4073 ctx->builder = builder = create_builder (ctx);
4074 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4076 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4078 /* Store the exception into the IL level exvar */
4079 if (bb->in_scount == 1) {
4080 g_assert (bb->in_scount == 1);
4081 exvar = bb->in_stack [0];
4083 // FIXME: This is shared with filter clauses ?
4084 g_assert (!values [exvar->dreg]);
4086 g_assert (ctx->ex_var);
4087 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4088 emit_volatile_store (ctx, exvar->dreg);
4094 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4096 MonoCompile *cfg = ctx->cfg;
4097 MonoMethodSignature *sig = ctx->sig;
4098 LLVMValueRef method = ctx->lmethod;
4099 LLVMValueRef *values = ctx->values;
4100 LLVMValueRef *addresses = ctx->addresses;
4101 LLVMCallInfo *linfo = ctx->linfo;
4102 BBInfo *bblocks = ctx->bblocks;
4104 LLVMBasicBlockRef cbb;
4105 LLVMBuilderRef builder, starting_builder;
4106 gboolean has_terminator;
4108 LLVMValueRef lhs, rhs;
4111 cbb = get_end_bb (ctx, bb);
4113 builder = create_builder (ctx);
4114 ctx->builder = builder;
4115 LLVMPositionBuilderAtEnd (builder, cbb);
4120 if (bb->flags & BB_EXCEPTION_HANDLER) {
4121 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4122 set_failure (ctx, "handler without invokes");
4127 emit_llvmonly_handler_start (ctx, bb, cbb);
4129 emit_handler_start (ctx, bb, builder);
4132 builder = ctx->builder;
4135 has_terminator = FALSE;
4136 starting_builder = builder;
4137 for (ins = bb->code; ins; ins = ins->next) {
4138 const char *spec = LLVM_INS_INFO (ins->opcode);
4140 char dname_buf [128];
4142 emit_dbg_loc (ctx, builder, ins->cil_code);
4147 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4148 * Start a new bblock. If the llvm optimization passes merge these, we
4149 * can work around that by doing a volatile load + cond branch from
4150 * localloc-ed memory.
4152 //set_failure (ctx, "basic block too long");
4153 cbb = gen_bb (ctx, "CONT_LONG_BB");
4154 LLVMBuildBr (ctx->builder, cbb);
4155 ctx->builder = builder = create_builder (ctx);
4156 LLVMPositionBuilderAtEnd (builder, cbb);
4157 ctx->bblocks [bb->block_num].end_bblock = cbb;
4162 /* There could be instructions after a terminator, skip them */
4165 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4166 sprintf (dname_buf, "t%d", ins->dreg);
4170 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4171 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4173 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4174 lhs = emit_volatile_load (ctx, ins->sreg1);
4176 /* It is ok for SETRET to have an uninitialized argument */
4177 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4178 set_failure (ctx, "sreg1");
4181 lhs = values [ins->sreg1];
4187 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4188 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4189 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4190 rhs = emit_volatile_load (ctx, ins->sreg2);
4192 if (!values [ins->sreg2]) {
4193 set_failure (ctx, "sreg2");
4196 rhs = values [ins->sreg2];
4202 //mono_print_ins (ins);
4203 switch (ins->opcode) {
4206 case OP_LIVERANGE_START:
4207 case OP_LIVERANGE_END:
4210 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4213 #if SIZEOF_VOID_P == 4
4214 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4216 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4220 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
4224 values [ins->dreg] = LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0);
4226 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
4228 case OP_DUMMY_ICONST:
4229 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4231 case OP_DUMMY_I8CONST:
4232 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4234 case OP_DUMMY_R8CONST:
4235 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4238 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4239 LLVMBuildBr (builder, target_bb);
4240 has_terminator = TRUE;
4247 LLVMBasicBlockRef new_bb;
4248 LLVMBuilderRef new_builder;
4250 // The default branch is already handled
4251 // FIXME: Handle it here
4253 /* Start new bblock */
4254 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4255 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4257 lhs = convert (ctx, lhs, LLVMInt32Type ());
4258 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4259 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4260 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4262 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4265 new_builder = create_builder (ctx);
4266 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4267 LLVMBuildUnreachable (new_builder);
4269 has_terminator = TRUE;
4270 g_assert (!ins->next);
4276 switch (linfo->ret.storage) {
4277 case LLVMArgVtypeInReg: {
4278 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4279 LLVMValueRef val, addr, retval;
4282 retval = LLVMGetUndef (ret_type);
4284 if (!addresses [ins->sreg1]) {
4286 * The return type is an LLVM vector type, have to convert between it and the
4287 * real return type which is a struct type.
4289 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret)));
4290 /* Convert to 2xi64 first */
4291 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4293 for (i = 0; i < 2; ++i) {
4294 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4295 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
4297 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4301 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
4302 for (i = 0; i < 2; ++i) {
4303 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4304 LLVMValueRef indexes [2], part_addr;
4306 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4307 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
4308 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
4310 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
4312 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4316 LLVMBuildRet (builder, retval);
4319 case LLVMArgVtypeAsScalar: {
4320 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4321 LLVMValueRef retval;
4323 g_assert (addresses [ins->sreg1]);
4325 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
4326 LLVMBuildRet (builder, retval);
4329 case LLVMArgVtypeByVal: {
4330 LLVMValueRef retval;
4332 g_assert (addresses [ins->sreg1]);
4333 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
4334 LLVMBuildRet (builder, retval);
4337 case LLVMArgVtypeByRef: {
4338 LLVMBuildRetVoid (builder);
4341 case LLVMArgGsharedvtFixed: {
4342 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
4343 /* The return value is in lhs, need to store to the vret argument */
4344 /* sreg1 might not be set */
4346 g_assert (cfg->vret_addr);
4347 g_assert (values [cfg->vret_addr->dreg]);
4348 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
4350 LLVMBuildRetVoid (builder);
4353 case LLVMArgGsharedvtFixedVtype: {
4355 LLVMBuildRetVoid (builder);
4358 case LLVMArgGsharedvtVariable: {
4360 LLVMBuildRetVoid (builder);
4363 case LLVMArgVtypeRetAddr: {
4364 LLVMBuildRetVoid (builder);
4367 case LLVMArgFpStruct: {
4368 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4369 LLVMValueRef retval;
4371 g_assert (addresses [ins->sreg1]);
4372 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
4373 LLVMBuildRet (builder, retval);
4377 case LLVMArgNormal: {
4378 if (!lhs || ctx->is_dead [ins->sreg1]) {
4380 * The method did not set its return value, probably because it
4381 * ends with a throw.
4384 LLVMBuildRetVoid (builder);
4386 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
4388 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
4390 has_terminator = TRUE;
4394 g_assert_not_reached ();
4403 case OP_ICOMPARE_IMM:
4404 case OP_LCOMPARE_IMM:
4405 case OP_COMPARE_IMM: {
4407 LLVMValueRef cmp, args [16];
4408 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
4410 if (ins->next->opcode == OP_NOP)
4413 if (ins->next->opcode == OP_BR)
4414 /* The comparison result is not needed */
4417 rel = mono_opcode_to_cond (ins->next->opcode);
4419 if (ins->opcode == OP_ICOMPARE_IMM) {
4420 lhs = convert (ctx, lhs, LLVMInt32Type ());
4421 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4423 if (ins->opcode == OP_LCOMPARE_IMM) {
4424 lhs = convert (ctx, lhs, LLVMInt64Type ());
4425 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4427 if (ins->opcode == OP_LCOMPARE) {
4428 lhs = convert (ctx, lhs, LLVMInt64Type ());
4429 rhs = convert (ctx, rhs, LLVMInt64Type ());
4431 if (ins->opcode == OP_ICOMPARE) {
4432 lhs = convert (ctx, lhs, LLVMInt32Type ());
4433 rhs = convert (ctx, rhs, LLVMInt32Type ());
4437 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4438 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
4439 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
4440 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
4443 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
4444 if (ins->opcode == OP_FCOMPARE) {
4445 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4446 } else if (ins->opcode == OP_RCOMPARE) {
4447 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4448 } else if (ins->opcode == OP_COMPARE_IMM) {
4449 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0)
4450 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
4452 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
4453 } else if (ins->opcode == OP_LCOMPARE_IMM) {
4454 if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) {
4455 /* The immediate is encoded in two fields */
4456 guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm);
4457 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), "");
4459 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), "");
4462 else if (ins->opcode == OP_COMPARE) {
4463 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
4464 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4466 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
4468 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4472 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
4473 cmp = LLVMBuildCall (ctx->builder, get_intrinsic (ctx, "llvm.expect.i1"), args, 2, "");
4476 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
4477 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
4479 * If the target bb contains PHI instructions, LLVM requires
4480 * two PHI entries for this bblock, while we only generate one.
4481 * So convert this to an unconditional bblock. (bxc #171).
4483 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
4485 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
4487 has_terminator = TRUE;
4488 } else if (MONO_IS_SETCC (ins->next)) {
4489 sprintf (dname_buf, "t%d", ins->next->dreg);
4491 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4493 /* Add stores for volatile variables */
4494 emit_volatile_store (ctx, ins->next->dreg);
4495 } else if (MONO_IS_COND_EXC (ins->next)) {
4496 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
4499 builder = ctx->builder;
4501 set_failure (ctx, "next");
4519 rel = mono_opcode_to_cond (ins->opcode);
4521 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4522 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4533 rel = mono_opcode_to_cond (ins->opcode);
4535 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4536 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4544 gboolean empty = TRUE;
4546 /* Check that all input bblocks really branch to us */
4547 for (i = 0; i < bb->in_count; ++i) {
4548 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4549 ins->inst_phi_args [i + 1] = -1;
4555 /* LLVM doesn't like phi instructions with zero operands */
4556 ctx->is_dead [ins->dreg] = TRUE;
4560 /* Created earlier, insert it now */
4561 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4563 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4564 int sreg1 = ins->inst_phi_args [i + 1];
4568 * Count the number of times the incoming bblock branches to us,
4569 * since llvm requires a separate entry for each.
4571 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4572 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4575 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4576 if (switch_ins->inst_many_bb [j] == bb)
4583 /* Remember for later */
4584 for (j = 0; j < count; ++j) {
4585 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4588 node->in_bb = bb->in_bb [i];
4590 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4600 values [ins->dreg] = lhs;
4604 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
4607 values [ins->dreg] = lhs;
4609 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
4611 * This is added by the spilling pass in case of the JIT,
4612 * but we have to do it ourselves.
4614 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
4618 case OP_MOVE_F_TO_I4: {
4619 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
4622 case OP_MOVE_I4_TO_F: {
4623 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
4626 case OP_MOVE_F_TO_I8: {
4627 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
4630 case OP_MOVE_I8_TO_F: {
4631 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
4664 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4665 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4667 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
4670 builder = ctx->builder;
4672 switch (ins->opcode) {
4675 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
4679 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
4683 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
4687 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
4691 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
4695 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
4699 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
4703 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4707 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
4711 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
4715 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
4719 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
4723 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
4727 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
4731 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4734 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4737 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4741 g_assert_not_reached ();
4748 lhs = convert (ctx, lhs, LLVMFloatType ());
4749 rhs = convert (ctx, rhs, LLVMFloatType ());
4750 switch (ins->opcode) {
4752 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4755 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4758 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4761 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4764 g_assert_not_reached ();
4773 case OP_IREM_UN_IMM:
4775 case OP_IDIV_UN_IMM:
4781 case OP_ISHR_UN_IMM:
4791 case OP_LSHR_UN_IMM:
4797 case OP_SHR_UN_IMM: {
4800 if (spec [MONO_INST_SRC1] == 'l') {
4801 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4803 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4806 emit_div_check (ctx, builder, bb, ins, lhs, imm);
4809 builder = ctx->builder;
4811 #if SIZEOF_VOID_P == 4
4812 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
4813 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4816 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4817 lhs = convert (ctx, lhs, IntPtrType ());
4818 imm = convert (ctx, imm, LLVMTypeOf (lhs));
4819 switch (ins->opcode) {
4823 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
4827 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
4832 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
4836 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
4838 case OP_IDIV_UN_IMM:
4839 case OP_LDIV_UN_IMM:
4840 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
4844 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
4846 case OP_IREM_UN_IMM:
4847 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
4852 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
4856 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
4860 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
4865 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
4870 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
4872 case OP_ISHR_UN_IMM:
4873 /* This is used to implement conv.u4, so the lhs could be an i8 */
4874 lhs = convert (ctx, lhs, LLVMInt32Type ());
4875 imm = convert (ctx, imm, LLVMInt32Type ());
4876 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4878 case OP_LSHR_UN_IMM:
4880 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4883 g_assert_not_reached ();
4888 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4891 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
4894 lhs = convert (ctx, lhs, LLVMDoubleType ());
4895 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
4898 lhs = convert (ctx, lhs, LLVMFloatType ());
4899 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
4902 guint32 v = 0xffffffff;
4903 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4907 guint64 v = 0xffffffffffffffffLL;
4908 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
4911 #if defined(TARGET_X86) || defined(TARGET_AMD64)
4913 LLVMValueRef v1, v2;
4915 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
4916 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
4917 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
4922 case OP_ICONV_TO_I1:
4923 case OP_ICONV_TO_I2:
4924 case OP_ICONV_TO_I4:
4925 case OP_ICONV_TO_U1:
4926 case OP_ICONV_TO_U2:
4927 case OP_ICONV_TO_U4:
4928 case OP_LCONV_TO_I1:
4929 case OP_LCONV_TO_I2:
4930 case OP_LCONV_TO_U1:
4931 case OP_LCONV_TO_U2:
4932 case OP_LCONV_TO_U4: {
4935 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
4937 /* Have to do two casts since our vregs have type int */
4938 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
4940 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
4942 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
4945 case OP_ICONV_TO_I8:
4946 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
4948 case OP_ICONV_TO_U8:
4949 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
4951 case OP_FCONV_TO_I4:
4952 case OP_RCONV_TO_I4:
4953 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
4955 case OP_FCONV_TO_I1:
4956 case OP_RCONV_TO_I1:
4957 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
4959 case OP_FCONV_TO_U1:
4960 case OP_RCONV_TO_U1:
4961 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
4963 case OP_FCONV_TO_I2:
4964 case OP_RCONV_TO_I2:
4965 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4967 case OP_FCONV_TO_U2:
4968 case OP_RCONV_TO_U2:
4969 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4971 case OP_RCONV_TO_U4:
4972 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
4974 case OP_FCONV_TO_I8:
4975 case OP_RCONV_TO_I8:
4976 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
4979 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
4981 case OP_ICONV_TO_R8:
4982 case OP_LCONV_TO_R8:
4983 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
4985 case OP_ICONV_TO_R_UN:
4986 case OP_LCONV_TO_R_UN:
4987 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
4989 #if SIZEOF_VOID_P == 4
4992 case OP_LCONV_TO_I4:
4993 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
4995 case OP_ICONV_TO_R4:
4996 case OP_LCONV_TO_R4:
4997 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
4999 values [ins->dreg] = v;
5001 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5003 case OP_FCONV_TO_R4:
5004 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5006 values [ins->dreg] = v;
5008 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5010 case OP_RCONV_TO_R8:
5011 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5013 case OP_RCONV_TO_R4:
5014 values [ins->dreg] = lhs;
5017 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5020 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5023 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5025 case OP_LOCALLOC_IMM: {
5028 guint32 size = ins->inst_imm;
5029 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5031 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5033 if (ins->flags & MONO_INST_INIT) {
5034 LLVMValueRef args [5];
5037 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5038 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
5039 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
5040 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5041 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memset.p0i8.i32"), args, 5, "");
5044 values [ins->dreg] = v;
5048 LLVMValueRef v, size;
5050 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5052 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5054 if (ins->flags & MONO_INST_INIT) {
5055 LLVMValueRef args [5];
5058 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5060 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
5061 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5062 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memset.p0i8.i32"), args, 5, "");
5064 values [ins->dreg] = v;
5068 case OP_LOADI1_MEMBASE:
5069 case OP_LOADU1_MEMBASE:
5070 case OP_LOADI2_MEMBASE:
5071 case OP_LOADU2_MEMBASE:
5072 case OP_LOADI4_MEMBASE:
5073 case OP_LOADU4_MEMBASE:
5074 case OP_LOADI8_MEMBASE:
5075 case OP_LOADR4_MEMBASE:
5076 case OP_LOADR8_MEMBASE:
5077 case OP_LOAD_MEMBASE:
5085 LLVMValueRef base, index, addr;
5087 gboolean sext = FALSE, zext = FALSE;
5088 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5090 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5095 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5096 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5101 if (ins->inst_offset == 0) {
5103 } else if (ins->inst_offset % size != 0) {
5104 /* Unaligned load */
5105 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5106 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5108 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5109 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5113 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5115 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, dname, is_volatile);
5117 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5119 * These will signal LLVM that these loads do not alias any stores, and
5120 * they can't fail, allowing them to be hoisted out of loops.
5122 set_invariant_load_flag (values [ins->dreg]);
5123 #if LLVM_API_VERSION < 100
5124 set_metadata_flag (values [ins->dreg], "mono.nofail.load");
5129 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5131 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5132 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5133 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5137 case OP_STOREI1_MEMBASE_REG:
5138 case OP_STOREI2_MEMBASE_REG:
5139 case OP_STOREI4_MEMBASE_REG:
5140 case OP_STOREI8_MEMBASE_REG:
5141 case OP_STORER4_MEMBASE_REG:
5142 case OP_STORER8_MEMBASE_REG:
5143 case OP_STORE_MEMBASE_REG: {
5145 LLVMValueRef index, addr;
5147 gboolean sext = FALSE, zext = FALSE;
5148 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5150 if (!values [ins->inst_destbasereg]) {
5151 set_failure (ctx, "inst_destbasereg");
5155 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5157 if (ins->inst_offset % size != 0) {
5158 /* Unaligned store */
5159 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5160 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5162 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5163 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5165 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
5169 case OP_STOREI1_MEMBASE_IMM:
5170 case OP_STOREI2_MEMBASE_IMM:
5171 case OP_STOREI4_MEMBASE_IMM:
5172 case OP_STOREI8_MEMBASE_IMM:
5173 case OP_STORE_MEMBASE_IMM: {
5175 LLVMValueRef index, addr;
5177 gboolean sext = FALSE, zext = FALSE;
5178 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5180 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5182 if (ins->inst_offset % size != 0) {
5183 /* Unaligned store */
5184 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5185 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5187 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5188 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5190 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
5195 emit_load (ctx, bb, &builder, sizeof (gpointer), convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
5197 case OP_OUTARG_VTRETADDR:
5205 case OP_VOIDCALL_MEMBASE:
5206 case OP_CALL_MEMBASE:
5207 case OP_LCALL_MEMBASE:
5208 case OP_FCALL_MEMBASE:
5209 case OP_RCALL_MEMBASE:
5210 case OP_VCALL_MEMBASE:
5211 case OP_VOIDCALL_REG:
5216 case OP_VCALL_REG: {
5217 process_call (ctx, bb, &builder, ins);
5222 LLVMValueRef indexes [2];
5223 MonoJumpInfo *tmp_ji, *ji;
5224 LLVMValueRef got_entry_addr;
5228 * FIXME: Can't allocate from the cfg mempool since that is freed if
5229 * the LLVM compile fails.
5231 tmp_ji = g_new0 (MonoJumpInfo, 1);
5232 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5233 tmp_ji->data.target = ins->inst_p0;
5235 ji = mono_aot_patch_info_dup (tmp_ji);
5238 ji->next = cfg->patch_info;
5239 cfg->patch_info = ji;
5241 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5242 got_offset = mono_aot_get_got_offset (cfg->patch_info);
5243 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5244 if (!mono_aot_is_shared_got_offset (got_offset)) {
5245 //mono_print_ji (ji);
5247 ctx->has_got_access = TRUE;
5250 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5251 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5252 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5254 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5255 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5257 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5258 if (!cfg->llvm_only)
5259 set_invariant_load_flag (values [ins->dreg]);
5262 case OP_NOT_REACHED:
5263 LLVMBuildUnreachable (builder);
5264 has_terminator = TRUE;
5265 g_assert (bb->block_num < cfg->max_block_num);
5266 ctx->unreachable [bb->block_num] = TRUE;
5267 /* Might have instructions after this */
5269 MonoInst *next = ins->next;
5271 * FIXME: If later code uses the regs defined by these instructions,
5272 * compilation will fail.
5274 MONO_DELETE_INS (bb, next);
5278 MonoInst *var = ins->inst_i0;
5280 if (var->opcode == OP_VTARG_ADDR) {
5281 /* The variable contains the vtype address */
5282 values [ins->dreg] = values [var->dreg];
5283 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
5284 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
5286 values [ins->dreg] = addresses [var->dreg];
5291 LLVMValueRef args [1];
5293 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5294 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.sin.f64"), args, 1, dname);
5298 LLVMValueRef args [1];
5300 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5301 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.cos.f64"), args, 1, dname);
5305 LLVMValueRef args [1];
5307 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5308 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.sqrt.f64"), args, 1, dname);
5312 LLVMValueRef args [1];
5314 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5315 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "fabs"), args, 1, dname);
5329 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5330 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5332 switch (ins->opcode) {
5335 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
5339 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
5343 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
5347 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
5350 g_assert_not_reached ();
5353 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
5356 case OP_ATOMIC_EXCHANGE_I4:
5357 case OP_ATOMIC_EXCHANGE_I8: {
5358 LLVMValueRef args [2];
5361 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
5362 t = LLVMInt32Type ();
5364 t = LLVMInt64Type ();
5366 g_assert (ins->inst_offset == 0);
5368 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5369 args [1] = convert (ctx, rhs, t);
5371 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
5374 case OP_ATOMIC_ADD_I4:
5375 case OP_ATOMIC_ADD_I8: {
5376 LLVMValueRef args [2];
5379 if (ins->opcode == OP_ATOMIC_ADD_I4)
5380 t = LLVMInt32Type ();
5382 t = LLVMInt64Type ();
5384 g_assert (ins->inst_offset == 0);
5386 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5387 args [1] = convert (ctx, rhs, t);
5388 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
5391 case OP_ATOMIC_CAS_I4:
5392 case OP_ATOMIC_CAS_I8: {
5393 LLVMValueRef args [3], val;
5396 if (ins->opcode == OP_ATOMIC_CAS_I4)
5397 t = LLVMInt32Type ();
5399 t = LLVMInt64Type ();
5401 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5403 args [1] = convert (ctx, values [ins->sreg3], t);
5405 args [2] = convert (ctx, values [ins->sreg2], t);
5406 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
5407 /* cmpxchg returns a pair */
5408 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
5411 case OP_MEMORY_BARRIER: {
5412 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
5415 case OP_ATOMIC_LOAD_I1:
5416 case OP_ATOMIC_LOAD_I2:
5417 case OP_ATOMIC_LOAD_I4:
5418 case OP_ATOMIC_LOAD_I8:
5419 case OP_ATOMIC_LOAD_U1:
5420 case OP_ATOMIC_LOAD_U2:
5421 case OP_ATOMIC_LOAD_U4:
5422 case OP_ATOMIC_LOAD_U8:
5423 case OP_ATOMIC_LOAD_R4:
5424 case OP_ATOMIC_LOAD_R8: {
5425 set_failure (ctx, "atomic mono.load intrinsic");
5429 gboolean sext, zext;
5431 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5432 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5433 LLVMValueRef index, addr;
5435 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5440 if (ins->inst_offset != 0) {
5441 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5442 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
5447 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5449 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, dname, is_volatile, barrier);
5452 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5454 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5458 case OP_ATOMIC_STORE_I1:
5459 case OP_ATOMIC_STORE_I2:
5460 case OP_ATOMIC_STORE_I4:
5461 case OP_ATOMIC_STORE_I8:
5462 case OP_ATOMIC_STORE_U1:
5463 case OP_ATOMIC_STORE_U2:
5464 case OP_ATOMIC_STORE_U4:
5465 case OP_ATOMIC_STORE_U8:
5466 case OP_ATOMIC_STORE_R4:
5467 case OP_ATOMIC_STORE_R8: {
5468 set_failure (ctx, "atomic mono.store intrinsic");
5472 gboolean sext, zext;
5474 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5475 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5476 LLVMValueRef index, addr, value;
5478 if (!values [ins->inst_destbasereg]) {
5479 set_failure (ctx, "inst_destbasereg");
5483 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5485 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5486 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5487 value = convert (ctx, values [ins->sreg1], t);
5489 emit_store_general (ctx, bb, &builder, size, value, addr, is_volatile, barrier);
5493 case OP_RELAXED_NOP: {
5494 #if defined(TARGET_AMD64) || defined(TARGET_X86)
5495 emit_call (ctx, bb, &builder, get_intrinsic (ctx, "llvm.x86.sse2.pause"), NULL, 0);
5502 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
5504 // 257 == FS segment register
5505 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
5507 // 256 == GS segment register
5508 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5511 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
5512 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
5513 /* See mono_amd64_emit_tls_get () */
5514 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
5516 // 256 == GS segment register
5517 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5518 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
5520 set_failure (ctx, "opcode tls-get");
5526 case OP_TLS_GET_REG: {
5527 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5528 /* See emit_tls_get_reg () */
5529 // 256 == GS segment register
5530 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5531 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt32Type ()), ptrtype, ""), "");
5533 set_failure (ctx, "opcode tls-get");
5539 case OP_TLS_SET_REG: {
5540 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5541 /* See emit_tls_get_reg () */
5542 // 256 == GS segment register
5543 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5544 LLVMBuildStore (builder, convert (ctx, lhs, IntPtrType ()), LLVMBuildIntToPtr (builder, convert (ctx, rhs, LLVMInt32Type ()), ptrtype, ""));
5546 set_failure (ctx, "opcode tls-set-reg");
5556 case OP_IADD_OVF_UN:
5558 case OP_ISUB_OVF_UN:
5560 case OP_IMUL_OVF_UN:
5561 #if SIZEOF_VOID_P == 8
5563 case OP_LADD_OVF_UN:
5565 case OP_LSUB_OVF_UN:
5567 case OP_LMUL_OVF_UN:
5570 LLVMValueRef args [2], val, ovf, func;
5572 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
5573 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
5574 func = get_intrinsic (ctx, ovf_op_to_intrins (ins->opcode));
5576 val = LLVMBuildCall (builder, func, args, 2, "");
5577 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
5578 ovf = LLVMBuildExtractValue (builder, val, 1, "");
5579 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
5582 builder = ctx->builder;
5588 * We currently model them using arrays. Promotion to local vregs is
5589 * disabled for them in mono_handle_global_vregs () in the LLVM case,
5590 * so we always have an entry in cfg->varinfo for them.
5591 * FIXME: Is this needed ?
5594 MonoClass *klass = ins->klass;
5595 LLVMValueRef args [5];
5599 set_failure (ctx, "!klass");
5603 if (!addresses [ins->dreg])
5604 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5605 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5606 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5607 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5609 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5610 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5611 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memset.p0i8.i32"), args, 5, "");
5614 case OP_DUMMY_VZERO:
5617 case OP_STOREV_MEMBASE:
5618 case OP_LOADV_MEMBASE:
5620 MonoClass *klass = ins->klass;
5621 LLVMValueRef src = NULL, dst, args [5];
5622 gboolean done = FALSE;
5626 set_failure (ctx, "!klass");
5630 if (mini_is_gsharedvt_klass (klass)) {
5632 set_failure (ctx, "gsharedvt");
5636 switch (ins->opcode) {
5637 case OP_STOREV_MEMBASE:
5638 if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg &&
5639 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
5640 /* Decomposed earlier */
5641 g_assert_not_reached ();
5644 if (!addresses [ins->sreg1]) {
5646 g_assert (values [ins->sreg1]);
5647 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
5648 LLVMBuildStore (builder, values [ins->sreg1], dst);
5651 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5652 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5655 case OP_LOADV_MEMBASE:
5656 if (!addresses [ins->dreg])
5657 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5658 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5659 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5662 if (!addresses [ins->sreg1])
5663 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
5664 if (!addresses [ins->dreg])
5665 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5666 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5667 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5670 g_assert_not_reached ();
5680 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5681 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5683 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5684 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5685 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memcpy.p0i8.p0i8.i32"), args, 5, "");
5688 case OP_LLVM_OUTARG_VT: {
5689 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
5690 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
5692 if (ainfo->storage == LLVMArgGsharedvtVariable) {
5693 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
5695 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
5696 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
5698 g_assert (addresses [ins->sreg1]);
5699 addresses [ins->dreg] = addresses [ins->sreg1];
5701 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
5702 if (!addresses [ins->sreg1]) {
5703 addresses [ins->sreg1] = build_alloca (ctx, t);
5704 g_assert (values [ins->sreg1]);
5706 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
5707 addresses [ins->dreg] = addresses [ins->sreg1];
5709 if (!addresses [ins->sreg1]) {
5710 addresses [ins->sreg1] = build_alloca (ctx, t);
5711 g_assert (values [ins->sreg1]);
5712 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
5714 addresses [ins->dreg] = addresses [ins->sreg1];
5722 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5724 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
5727 case OP_LOADX_MEMBASE: {
5728 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
5731 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5732 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
5735 case OP_STOREX_MEMBASE: {
5736 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
5739 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5740 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
5747 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
5751 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
5757 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
5761 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
5765 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
5769 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
5772 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
5775 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
5778 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
5782 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
5793 LLVMValueRef v = NULL;
5795 switch (ins->opcode) {
5800 t = LLVMVectorType (LLVMInt32Type (), 4);
5801 rt = LLVMVectorType (LLVMFloatType (), 4);
5807 t = LLVMVectorType (LLVMInt64Type (), 2);
5808 rt = LLVMVectorType (LLVMDoubleType (), 2);
5811 t = LLVMInt32Type ();
5812 rt = LLVMInt32Type ();
5813 g_assert_not_reached ();
5816 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5817 rhs = LLVMBuildBitCast (builder, rhs, t, "");
5818 switch (ins->opcode) {
5821 v = LLVMBuildAnd (builder, lhs, rhs, "");
5825 v = LLVMBuildOr (builder, lhs, rhs, "");
5829 v = LLVMBuildXor (builder, lhs, rhs, "");
5833 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
5836 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
5860 case OP_PADDB_SAT_UN:
5861 case OP_PADDW_SAT_UN:
5862 case OP_PSUBB_SAT_UN:
5863 case OP_PSUBW_SAT_UN:
5871 case OP_PMULW_HIGH_UN: {
5872 LLVMValueRef args [2];
5877 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5884 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, ""), LLVMTypeOf (lhs), "");
5888 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, ""), LLVMTypeOf (lhs), "");
5896 case OP_EXTRACTX_U2:
5898 case OP_EXTRACT_U1: {
5900 gboolean zext = FALSE;
5902 t = simd_op_to_llvm_type (ins->opcode);
5904 switch (ins->opcode) {
5912 case OP_EXTRACTX_U2:
5917 t = LLVMInt32Type ();
5918 g_assert_not_reached ();
5921 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5922 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
5924 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
5933 case OP_EXPAND_R8: {
5934 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
5935 LLVMValueRef mask [16], v;
5938 for (i = 0; i < 16; ++i)
5939 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5941 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
5943 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5944 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
5949 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5952 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5955 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5958 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5961 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5964 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5975 case OP_EXTRACT_MASK:
5982 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
5984 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
5990 LLVMValueRef args [3];
5994 args [2] = LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE);
5996 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 3, dname);
6001 /* This is only used for implementing shifts by non-immediate */
6002 values [ins->dreg] = lhs;
6013 LLVMValueRef args [3];
6016 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
6018 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6029 case OP_PSHLQ_REG: {
6030 LLVMValueRef args [3];
6033 args [1] = values [ins->sreg2];
6035 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6042 case OP_PSHUFLEW_LOW:
6043 case OP_PSHUFLEW_HIGH: {
6045 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
6046 int i, mask_size = 0;
6047 int imask = ins->inst_c0;
6049 /* Convert the x86 shuffle mask to LLVM's */
6050 switch (ins->opcode) {
6053 mask [0] = ((imask >> 0) & 3);
6054 mask [1] = ((imask >> 2) & 3);
6055 mask [2] = ((imask >> 4) & 3) + 4;
6056 mask [3] = ((imask >> 6) & 3) + 4;
6057 v1 = values [ins->sreg1];
6058 v2 = values [ins->sreg2];
6062 mask [0] = ((imask >> 0) & 1);
6063 mask [1] = ((imask >> 1) & 1) + 2;
6064 v1 = values [ins->sreg1];
6065 v2 = values [ins->sreg2];
6067 case OP_PSHUFLEW_LOW:
6069 mask [0] = ((imask >> 0) & 3);
6070 mask [1] = ((imask >> 2) & 3);
6071 mask [2] = ((imask >> 4) & 3);
6072 mask [3] = ((imask >> 6) & 3);
6077 v1 = values [ins->sreg1];
6078 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6080 case OP_PSHUFLEW_HIGH:
6086 mask [4] = 4 + ((imask >> 0) & 3);
6087 mask [5] = 4 + ((imask >> 2) & 3);
6088 mask [6] = 4 + ((imask >> 4) & 3);
6089 mask [7] = 4 + ((imask >> 6) & 3);
6090 v1 = values [ins->sreg1];
6091 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6095 mask [0] = ((imask >> 0) & 3);
6096 mask [1] = ((imask >> 2) & 3);
6097 mask [2] = ((imask >> 4) & 3);
6098 mask [3] = ((imask >> 6) & 3);
6099 v1 = values [ins->sreg1];
6100 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6103 g_assert_not_reached ();
6105 for (i = 0; i < mask_size; ++i)
6106 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6108 values [ins->dreg] =
6109 LLVMBuildShuffleVector (builder, v1, v2,
6110 LLVMConstVector (mask_values, mask_size), dname);
6114 case OP_UNPACK_LOWB:
6115 case OP_UNPACK_LOWW:
6116 case OP_UNPACK_LOWD:
6117 case OP_UNPACK_LOWQ:
6118 case OP_UNPACK_LOWPS:
6119 case OP_UNPACK_LOWPD:
6120 case OP_UNPACK_HIGHB:
6121 case OP_UNPACK_HIGHW:
6122 case OP_UNPACK_HIGHD:
6123 case OP_UNPACK_HIGHQ:
6124 case OP_UNPACK_HIGHPS:
6125 case OP_UNPACK_HIGHPD: {
6127 LLVMValueRef mask_values [16];
6128 int i, mask_size = 0;
6129 gboolean low = FALSE;
6131 switch (ins->opcode) {
6132 case OP_UNPACK_LOWB:
6136 case OP_UNPACK_LOWW:
6140 case OP_UNPACK_LOWD:
6141 case OP_UNPACK_LOWPS:
6145 case OP_UNPACK_LOWQ:
6146 case OP_UNPACK_LOWPD:
6150 case OP_UNPACK_HIGHB:
6153 case OP_UNPACK_HIGHW:
6156 case OP_UNPACK_HIGHD:
6157 case OP_UNPACK_HIGHPS:
6160 case OP_UNPACK_HIGHQ:
6161 case OP_UNPACK_HIGHPD:
6165 g_assert_not_reached ();
6169 for (i = 0; i < (mask_size / 2); ++i) {
6171 mask [(i * 2) + 1] = mask_size + i;
6174 for (i = 0; i < (mask_size / 2); ++i) {
6175 mask [(i * 2)] = (mask_size / 2) + i;
6176 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
6180 for (i = 0; i < mask_size; ++i)
6181 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6183 values [ins->dreg] =
6184 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
6185 LLVMConstVector (mask_values, mask_size), dname);
6190 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6191 LLVMValueRef v, val;
6193 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6194 val = LLVMConstNull (t);
6195 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6196 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
6198 values [ins->dreg] = val;
6202 case OP_DUPPS_HIGH: {
6203 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6204 LLVMValueRef v1, v2, val;
6207 if (ins->opcode == OP_DUPPS_LOW) {
6208 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6209 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6211 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6212 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6214 val = LLVMConstNull (t);
6215 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6216 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6217 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6218 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6220 values [ins->dreg] = val;
6230 * EXCEPTION HANDLING
6232 case OP_IMPLICIT_EXCEPTION:
6233 /* This marks a place where an implicit exception can happen */
6234 if (bb->region != -1)
6235 set_failure (ctx, "implicit-exception");
6239 gboolean rethrow = (ins->opcode == OP_RETHROW);
6240 if (ctx->llvm_only) {
6241 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
6242 has_terminator = TRUE;
6243 ctx->unreachable [bb->block_num] = TRUE;
6245 emit_throw (ctx, bb, rethrow, lhs);
6246 builder = ctx->builder;
6250 case OP_CALL_HANDLER: {
6252 * We don't 'call' handlers, but instead simply branch to them.
6253 * The code generated by ENDFINALLY will branch back to us.
6255 LLVMBasicBlockRef noex_bb;
6257 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
6259 bb_list = info->call_handler_return_bbs;
6262 * Set the indicator variable for the finally clause.
6264 lhs = info->finally_ind;
6266 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
6268 /* Branch to the finally clause */
6269 LLVMBuildBr (builder, info->call_handler_target_bb);
6271 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
6272 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
6274 builder = ctx->builder = create_builder (ctx);
6275 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
6277 bblocks [bb->block_num].end_bblock = noex_bb;
6280 case OP_START_HANDLER: {
6283 case OP_ENDFINALLY: {
6284 LLVMBasicBlockRef resume_bb;
6285 MonoBasicBlock *handler_bb;
6286 LLVMValueRef val, switch_ins, callee;
6290 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
6291 g_assert (handler_bb);
6292 info = &bblocks [handler_bb->block_num];
6293 lhs = info->finally_ind;
6296 bb_list = info->call_handler_return_bbs;
6298 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
6300 /* Load the finally variable */
6301 val = LLVMBuildLoad (builder, lhs, "");
6303 /* Reset the variable */
6304 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
6306 /* Branch to either resume_bb, or to the bblocks in bb_list */
6307 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
6309 * The other targets are added at the end to handle OP_CALL_HANDLER
6310 * opcodes processed later.
6312 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
6314 builder = ctx->builder = create_builder (ctx);
6315 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
6317 if (ctx->llvm_only) {
6318 emit_resume_eh (ctx, bb);
6320 if (ctx->cfg->compile_aot) {
6321 callee = get_callee (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline");
6323 #if LLVM_API_VERSION > 100
6324 MonoJitICallInfo *info;
6326 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
6328 gpointer target = (void*)info->func;
6329 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
6330 callee = emit_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, target);
6332 callee = LLVMGetNamedFunction (ctx->lmodule, "llvm_resume_unwind_trampoline");
6335 LLVMBuildCall (builder, callee, NULL, 0, "");
6336 LLVMBuildUnreachable (builder);
6339 has_terminator = TRUE;
6342 case OP_IL_SEQ_POINT:
6347 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
6348 set_failure (ctx, reason);
6356 /* Convert the value to the type required by phi nodes */
6357 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
6358 if (!values [ins->dreg])
6360 values [ins->dreg] = addresses [ins->dreg];
6362 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
6365 /* Add stores for volatile variables */
6366 if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
6367 emit_volatile_store (ctx, ins->dreg);
6373 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
6374 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
6377 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
6378 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
6379 LLVMBuildRetVoid (builder);
6382 if (bb == cfg->bb_entry)
6383 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
6387 * mono_llvm_check_method_supported:
6389 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
6390 * compiling a method twice.
6393 mono_llvm_check_method_supported (MonoCompile *cfg)
6400 if (cfg->method->save_lmf) {
6401 cfg->exception_message = g_strdup ("lmf");
6402 cfg->disable_llvm = TRUE;
6404 if (cfg->disable_llvm)
6408 * Nested clauses where one of the clauses is a finally clause is
6409 * not supported, because LLVM can't figure out the control flow,
6410 * probably because we resume exception handling by calling our
6411 * own function instead of using the 'resume' llvm instruction.
6413 for (i = 0; i < cfg->header->num_clauses; ++i) {
6414 for (j = 0; j < cfg->header->num_clauses; ++j) {
6415 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
6416 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
6418 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
6419 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
6420 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
6421 cfg->exception_message = g_strdup ("nested clauses");
6422 cfg->disable_llvm = TRUE;
6427 if (cfg->disable_llvm)
6431 if (cfg->method->dynamic) {
6432 cfg->exception_message = g_strdup ("dynamic.");
6433 cfg->disable_llvm = TRUE;
6435 if (cfg->disable_llvm)
6439 static LLVMCallInfo*
6440 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
6442 LLVMCallInfo *linfo;
6445 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
6449 * Gsharedvt methods have the following calling convention:
6450 * - all arguments are passed by ref, even non generic ones
6451 * - the return value is returned by ref too, using a vret
6452 * argument passed after 'this'.
6454 n = sig->param_count + sig->hasthis;
6455 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
6459 linfo->args [pindex ++].storage = LLVMArgNormal;
6461 if (sig->ret->type != MONO_TYPE_VOID) {
6462 if (mini_is_gsharedvt_variable_type (sig->ret))
6463 linfo->ret.storage = LLVMArgGsharedvtVariable;
6464 else if (mini_type_is_vtype (sig->ret))
6465 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
6467 linfo->ret.storage = LLVMArgGsharedvtFixed;
6468 linfo->vret_arg_index = pindex;
6470 linfo->ret.storage = LLVMArgNone;
6473 for (i = 0; i < sig->param_count; ++i) {
6474 if (sig->params [i]->byref)
6475 linfo->args [pindex].storage = LLVMArgNormal;
6476 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
6477 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
6478 else if (mini_type_is_vtype (sig->params [i]))
6479 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
6481 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
6482 linfo->args [pindex].type = sig->params [i];
6489 linfo = mono_arch_get_llvm_call_info (cfg, sig);
6490 for (i = 0; i < sig->param_count; ++i)
6491 linfo->args [i + sig->hasthis].type = sig->params [i];
6497 emit_method_inner (EmitContext *ctx);
6500 free_ctx (EmitContext *ctx)
6504 g_free (ctx->values);
6505 g_free (ctx->addresses);
6506 g_free (ctx->vreg_types);
6507 g_free (ctx->vreg_cli_types);
6508 g_free (ctx->is_dead);
6509 g_free (ctx->unreachable);
6510 g_ptr_array_free (ctx->phi_values, TRUE);
6511 g_free (ctx->bblocks);
6512 g_hash_table_destroy (ctx->region_to_handler);
6513 g_hash_table_destroy (ctx->clause_to_handler);
6514 g_hash_table_destroy (ctx->jit_callees);
6515 g_free (ctx->method_name);
6516 g_ptr_array_free (ctx->bblock_list, TRUE);
6518 for (l = ctx->builders; l; l = l->next) {
6519 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
6520 LLVMDisposeBuilder (builder);
6527 * mono_llvm_emit_method:
6529 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
6532 mono_llvm_emit_method (MonoCompile *cfg)
6536 gboolean is_linkonce = FALSE;
6539 /* The code below might acquire the loader lock, so use it for global locking */
6540 mono_loader_lock ();
6542 /* Used to communicate with the callbacks */
6543 mono_native_tls_set_value (current_cfg_tls_id, cfg);
6545 ctx = g_new0 (EmitContext, 1);
6547 ctx->mempool = cfg->mempool;
6550 * This maps vregs to the LLVM instruction defining them
6552 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
6554 * This maps vregs for volatile variables to the LLVM instruction defining their
6557 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
6558 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
6559 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
6560 ctx->phi_values = g_ptr_array_sized_new (256);
6562 * This signals whenever the vreg was defined by a phi node with no input vars
6563 * (i.e. all its input bblocks end with NOT_REACHABLE).
6565 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
6566 /* Whenever the bblock is unreachable */
6567 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
6568 ctx->bblock_list = g_ptr_array_sized_new (256);
6570 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
6571 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
6572 ctx->method_to_callers = g_hash_table_new (NULL, NULL);
6573 ctx->jit_callees = g_hash_table_new (NULL, NULL);
6574 if (cfg->compile_aot) {
6575 ctx->module = &aot_module;
6579 * Allow the linker to discard duplicate copies of wrappers, generic instances etc. by using the 'linkonce'
6580 * linkage for them. This requires the following:
6581 * - the method needs to have a unique mangled name
6582 * - llvmonly mode, since the code in aot-runtime.c would initialize got slots in the wrong aot image etc.
6584 is_linkonce = ctx->module->llvm_only && ctx->module->static_link && mono_aot_is_linkonce_method (cfg->method);
6586 method_name = mono_aot_get_mangled_method_name (cfg->method);
6588 is_linkonce = FALSE;
6591 printf ("%s %s\n", mono_method_full_name (cfg->method, 1), method_name);
6593 printf ("%s\n", mono_method_full_name (cfg->method, 1));
6597 method_name = mono_aot_get_method_name (cfg);
6598 cfg->llvm_method_name = g_strdup (method_name);
6600 init_jit_module (cfg->domain);
6601 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
6602 method_name = mono_method_full_name (cfg->method, TRUE);
6604 ctx->method_name = method_name;
6605 ctx->is_linkonce = is_linkonce;
6607 #if LLVM_API_VERSION > 100
6608 ctx->lmodule = LLVMModuleCreateWithName ("jit-module");
6610 ctx->lmodule = ctx->module->lmodule;
6612 ctx->llvm_only = ctx->module->llvm_only;
6614 emit_method_inner (ctx);
6616 if (!ctx_ok (ctx)) {
6618 /* Need to add unused phi nodes as they can be referenced by other values */
6619 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
6620 LLVMBuilderRef builder;
6622 builder = create_builder (ctx);
6623 LLVMPositionBuilderAtEnd (builder, phi_bb);
6625 for (i = 0; i < ctx->phi_values->len; ++i) {
6626 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
6627 if (LLVMGetInstructionParent (v) == NULL)
6628 LLVMInsertIntoBuilder (builder, v);
6631 LLVMDeleteFunction (ctx->lmethod);
6637 mono_native_tls_set_value (current_cfg_tls_id, NULL);
6639 mono_loader_unlock ();
6643 emit_method_inner (EmitContext *ctx)
6645 MonoCompile *cfg = ctx->cfg;
6646 MonoMethodSignature *sig;
6648 LLVMTypeRef method_type;
6649 LLVMValueRef method = NULL;
6650 LLVMValueRef *values = ctx->values;
6651 int i, max_block_num, bb_index;
6652 gboolean last = FALSE;
6653 LLVMCallInfo *linfo;
6654 LLVMModuleRef lmodule = ctx->lmodule;
6656 GPtrArray *bblock_list = ctx->bblock_list;
6657 MonoMethodHeader *header;
6658 MonoExceptionClause *clause;
6661 if (cfg->gsharedvt && !cfg->llvm_only) {
6662 set_failure (ctx, "gsharedvt");
6668 static int count = 0;
6671 if (g_getenv ("LLVM_COUNT")) {
6672 if (count == atoi (g_getenv ("LLVM_COUNT"))) {
6673 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
6677 if (count > atoi (g_getenv ("LLVM_COUNT"))) {
6678 set_failure (ctx, "count");
6685 sig = mono_method_signature (cfg->method);
6688 linfo = get_llvm_call_info (cfg, sig);
6694 linfo->rgctx_arg = TRUE;
6695 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
6699 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
6700 ctx->lmethod = method;
6702 if (!cfg->llvm_only)
6703 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
6704 LLVMSetLinkage (method, LLVMPrivateLinkage);
6706 LLVMAddFunctionAttr (method, LLVMUWTable);
6708 if (cfg->compile_aot) {
6709 LLVMSetLinkage (method, LLVMInternalLinkage);
6710 if (ctx->module->external_symbols) {
6711 LLVMSetLinkage (method, LLVMExternalLinkage);
6712 LLVMSetVisibility (method, LLVMHiddenVisibility);
6714 if (ctx->is_linkonce) {
6715 LLVMSetLinkage (method, LLVMLinkOnceAnyLinkage);
6716 LLVMSetVisibility (method, LLVMDefaultVisibility);
6719 #if LLVM_API_VERSION > 100
6720 LLVMSetLinkage (method, LLVMExternalLinkage);
6722 LLVMSetLinkage (method, LLVMPrivateLinkage);
6726 if (cfg->method->save_lmf && !cfg->llvm_only) {
6727 set_failure (ctx, "lmf");
6731 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
6732 set_failure (ctx, "pinvoke signature");
6736 header = cfg->header;
6737 for (i = 0; i < header->num_clauses; ++i) {
6738 clause = &header->clauses [i];
6739 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
6740 set_failure (ctx, "non-finally/catch clause.");
6744 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
6745 /* We can't handle inlined methods with clauses */
6746 LLVMAddFunctionAttr (method, LLVMNoInlineAttribute);
6748 if (linfo->rgctx_arg) {
6749 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
6750 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
6752 * We mark the rgctx parameter with the inreg attribute, which is mapped to
6753 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
6754 * CC_X86_64_Mono in X86CallingConv.td.
6756 if (!ctx->llvm_only)
6757 LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute);
6758 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
6760 ctx->rgctx_arg_pindex = -1;
6762 if (cfg->vret_addr) {
6763 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
6764 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
6765 if (linfo->ret.storage == LLVMArgVtypeByRef) {
6766 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMStructRetAttribute);
6767 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
6772 ctx->this_arg_pindex = linfo->this_arg_pindex;
6773 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
6774 values [cfg->args [0]->dreg] = ctx->this_arg;
6775 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
6778 names = g_new (char *, sig->param_count);
6779 mono_method_get_param_names (cfg->method, (const char **) names);
6781 for (i = 0; i < sig->param_count; ++i) {
6782 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
6784 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
6787 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
6788 name = g_strdup_printf ("dummy_%d_%d", i, j);
6789 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
6793 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
6794 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
6795 if (names [i] && names [i][0] != '\0')
6796 name = g_strdup_printf ("p_arg_%s", names [i]);
6798 name = g_strdup_printf ("p_arg_%d", i);
6800 if (names [i] && names [i][0] != '\0')
6801 name = g_strdup_printf ("arg_%s", names [i]);
6803 name = g_strdup_printf ("arg_%d", i);
6805 LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name);
6807 if (ainfo->storage == LLVMArgVtypeByVal)
6808 LLVMAddAttribute (LLVMGetParam (method, pindex), LLVMByValAttribute);
6810 if (ainfo->storage == LLVMArgVtypeByRef) {
6812 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
6817 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
6818 ctx->minfo = mono_debug_lookup_method (cfg->method);
6819 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
6823 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
6824 max_block_num = MAX (max_block_num, bb->block_num);
6825 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
6827 /* Add branches between non-consecutive bblocks */
6828 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6829 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
6830 bb->next_bb != bb->last_ins->inst_false_bb) {
6832 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
6833 inst->opcode = OP_BR;
6834 inst->inst_target_bb = bb->last_ins->inst_false_bb;
6835 mono_bblock_add_inst (bb, inst);
6840 * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR
6841 * was later optimized away, so clear these flags, and add them back for the still
6842 * present OP_LDADDR instructions.
6844 for (i = 0; i < cfg->next_vreg; ++i) {
6847 ins = get_vreg_to_inst (cfg, i);
6848 if (ins && ins != cfg->rgctx_var)
6849 ins->flags &= ~MONO_INST_INDIRECT;
6853 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
6855 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6857 LLVMBuilderRef builder;
6859 char dname_buf[128];
6861 builder = create_builder (ctx);
6863 for (ins = bb->code; ins; ins = ins->next) {
6864 switch (ins->opcode) {
6869 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, &ins->klass->byval_arg));
6874 if (ins->opcode == OP_VPHI) {
6875 /* Treat valuetype PHI nodes as operating on the address itself */
6876 g_assert (ins->klass);
6877 phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
6881 * Have to precreate these, as they can be referenced by
6882 * earlier instructions.
6884 sprintf (dname_buf, "t%d", ins->dreg);
6886 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
6888 if (ins->opcode == OP_VPHI)
6889 ctx->addresses [ins->dreg] = values [ins->dreg];
6891 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
6894 * Set the expected type of the incoming arguments since these have
6895 * to have the same type.
6897 for (i = 0; i < ins->inst_phi_args [0]; i++) {
6898 int sreg1 = ins->inst_phi_args [i + 1];
6901 ctx->vreg_types [sreg1] = phi_type;
6906 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
6915 * Create an ordering for bblocks, use the depth first order first, then
6916 * put the exception handling bblocks last.
6918 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
6919 bb = cfg->bblocks [bb_index];
6920 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
6921 g_ptr_array_add (bblock_list, bb);
6922 bblocks [bb->block_num].added = TRUE;
6926 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6927 if (!bblocks [bb->block_num].added)
6928 g_ptr_array_add (bblock_list, bb);
6932 * Second pass: generate code.
6935 LLVMBuilderRef entry_builder = create_builder (ctx);
6936 LLVMBasicBlockRef entry_bb = get_bb (ctx, cfg->bb_entry);
6937 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
6938 emit_entry_bb (ctx, entry_builder);
6940 // Make landing pads first
6941 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
6943 if (ctx->llvm_only) {
6944 size_t group_index = 0;
6945 while (group_index < cfg->header->num_clauses) {
6947 size_t cursor = group_index;
6948 while (cursor < cfg->header->num_clauses &&
6949 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
6950 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
6955 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
6956 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
6957 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
6959 group_index = cursor;
6963 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
6964 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
6966 // Prune unreachable mono BBs.
6967 if (!(bb == cfg->bb_entry || bb->in_count > 0))
6970 process_bb (ctx, bb);
6974 g_hash_table_destroy (ctx->exc_meta);
6976 mono_memory_barrier ();
6978 /* Add incoming phi values */
6979 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6980 GSList *l, *ins_list;
6982 ins_list = bblocks [bb->block_num].phi_nodes;
6984 for (l = ins_list; l; l = l->next) {
6985 PhiNode *node = (PhiNode*)l->data;
6986 MonoInst *phi = node->phi;
6987 int sreg1 = node->sreg;
6988 LLVMBasicBlockRef in_bb;
6993 in_bb = get_end_bb (ctx, node->in_bb);
6995 if (ctx->unreachable [node->in_bb->block_num])
6998 if (!values [sreg1]) {
6999 /* Can happen with values in EH clauses */
7000 set_failure (ctx, "incoming phi sreg1");
7004 if (phi->opcode == OP_VPHI) {
7005 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
7006 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
7008 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
7009 set_failure (ctx, "incoming phi arg type mismatch");
7012 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
7013 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
7018 /* Nullify empty phi instructions */
7019 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7020 GSList *l, *ins_list;
7022 ins_list = bblocks [bb->block_num].phi_nodes;
7024 for (l = ins_list; l; l = l->next) {
7025 PhiNode *node = (PhiNode*)l->data;
7026 MonoInst *phi = node->phi;
7027 LLVMValueRef phi_ins = values [phi->dreg];
7030 /* Already removed */
7033 if (LLVMCountIncoming (phi_ins) == 0) {
7034 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
7035 LLVMInstructionEraseFromParent (phi_ins);
7036 values [phi->dreg] = NULL;
7041 /* Create the SWITCH statements for ENDFINALLY instructions */
7042 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7043 BBInfo *info = &bblocks [bb->block_num];
7045 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
7046 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
7047 GSList *bb_list = info->call_handler_return_bbs;
7049 for (i = 0; i < g_slist_length (bb_list); ++i)
7050 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)(g_slist_nth (bb_list, i)->data));
7054 /* Initialize the method if needed */
7055 if (cfg->compile_aot && ctx->llvm_only) {
7056 // FIXME: Add more shared got entries
7057 ctx->builder = create_builder (ctx);
7058 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
7060 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
7062 // FIXME: beforefieldinit
7063 if (ctx->has_got_access || mono_class_get_cctor (cfg->method->klass)) {
7064 emit_init_method (ctx);
7066 LLVMBuildBr (ctx->builder, ctx->inited_bb);
7070 if (cfg->llvm_only) {
7071 GHashTableIter iter;
7073 GSList *callers, *l, *l2;
7076 * Add the contents of ctx->method_to_callers to module->method_to_callers.
7077 * We can't do this earlier, as it contains llvm instructions which can be
7078 * freed if compilation fails.
7079 * FIXME: Get rid of this when all methods can be llvm compiled.
7081 g_hash_table_iter_init (&iter, ctx->method_to_callers);
7082 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
7083 for (l = callers; l; l = l->next) {
7084 l2 = (GSList*)g_hash_table_lookup (ctx->module->method_to_callers, method);
7085 l2 = g_slist_prepend (l2, l->data);
7086 g_hash_table_insert (ctx->module->method_to_callers, method, l2);
7091 if (cfg->verbose_level > 1)
7092 mono_llvm_dump_value (method);
7094 if (cfg->compile_aot && !cfg->llvm_only)
7095 mark_as_used (ctx->module, method);
7097 if (cfg->compile_aot && !cfg->llvm_only) {
7098 LLVMValueRef md_args [16];
7099 LLVMValueRef md_node;
7102 method_index = mono_aot_get_method_index (cfg->orig_method);
7103 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
7104 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
7105 md_node = LLVMMDNode (md_args, 2);
7106 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
7107 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
7110 if (cfg->compile_aot) {
7111 /* Don't generate native code, keep the LLVM IR */
7112 if (cfg->verbose_level)
7113 printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), ctx->method_name);
7115 #if LLVM_API_VERSION < 100
7116 /* VerifyFunction can't handle some of the debug info created by DIBuilder in llvm 3.9 */
7117 int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
7118 g_assert (err == 0);
7121 //LLVMVerifyFunction(method, 0);
7122 #if LLVM_API_VERSION > 100
7123 MonoDomain *domain = mono_domain_get ();
7124 MonoJitDomainInfo *domain_info;
7125 int nvars = g_hash_table_size (ctx->jit_callees);
7126 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
7127 gpointer *callee_addrs = g_new0 (gpointer, nvars);
7128 GHashTableIter iter;
7134 * Compute the addresses of the LLVM globals pointing to the
7135 * methods called by the current method. Pass it to the trampoline
7136 * code so it can update them after their corresponding method was
7139 g_hash_table_iter_init (&iter, ctx->jit_callees);
7141 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
7142 callee_vars [i ++] = var;
7144 cfg->native_code = mono_llvm_compile_method (ctx->module->mono_ee, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
7146 decode_llvm_eh_info (ctx, eh_frame);
7148 mono_domain_lock (domain);
7149 domain_info = domain_jit_info (domain);
7150 if (!domain_info->llvm_jit_callees)
7151 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
7152 g_hash_table_iter_init (&iter, ctx->jit_callees);
7154 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
7155 GSList *addrs = g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
7156 addrs = g_slist_prepend (addrs, callee_addrs [i]);
7157 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
7160 mono_domain_unlock (domain);
7162 mono_llvm_optimize_method (ctx->module->mono_ee, ctx->lmethod);
7164 if (cfg->verbose_level > 1)
7165 mono_llvm_dump_value (ctx->lmethod);
7167 cfg->native_code = (unsigned char*)LLVMGetPointerToGlobal (ctx->module->ee, ctx->lmethod);
7169 /* Set by emit_cb */
7170 g_assert (cfg->code_len);
7174 if (ctx->module->method_to_lmethod)
7175 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
7176 if (ctx->module->idx_to_lmethod)
7177 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
7179 if (ctx->llvm_only && cfg->orig_method->klass->valuetype && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
7180 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
7184 * mono_llvm_create_vars:
7186 * Same as mono_arch_create_vars () for LLVM.
7189 mono_llvm_create_vars (MonoCompile *cfg)
7191 MonoMethodSignature *sig;
7193 sig = mono_method_signature (cfg->method);
7194 if (cfg->gsharedvt && cfg->llvm_only) {
7195 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
7196 cfg->vret_addr = mono_compile_create_var (cfg, &mono_get_intptr_class ()->byval_arg, OP_ARG);
7197 if (G_UNLIKELY (cfg->verbose_level > 1)) {
7198 printf ("vret_addr = ");
7199 mono_print_ins (cfg->vret_addr);
7203 mono_arch_create_vars (cfg);
7208 * mono_llvm_emit_call:
7210 * Same as mono_arch_emit_call () for LLVM.
7213 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
7216 MonoMethodSignature *sig;
7217 int i, n, stack_size;
7222 sig = call->signature;
7223 n = sig->param_count + sig->hasthis;
7225 call->cinfo = get_llvm_call_info (cfg, sig);
7227 if (cfg->disable_llvm)
7230 if (sig->call_convention == MONO_CALL_VARARG) {
7231 cfg->exception_message = g_strdup ("varargs");
7232 cfg->disable_llvm = TRUE;
7235 for (i = 0; i < n; ++i) {
7238 ainfo = call->cinfo->args + i;
7240 in = call->args [i];
7242 /* Simply remember the arguments */
7243 switch (ainfo->storage) {
7244 case LLVMArgNormal: {
7245 MonoType *t = (sig->hasthis && i == 0) ? &mono_get_intptr_class ()->byval_arg : ainfo->type;
7248 opcode = mono_type_to_regmove (cfg, t);
7249 if (opcode == OP_FMOVE) {
7250 MONO_INST_NEW (cfg, ins, OP_FMOVE);
7251 ins->dreg = mono_alloc_freg (cfg);
7252 } else if (opcode == OP_LMOVE) {
7253 MONO_INST_NEW (cfg, ins, OP_LMOVE);
7254 ins->dreg = mono_alloc_lreg (cfg);
7255 } else if (opcode == OP_RMOVE) {
7256 MONO_INST_NEW (cfg, ins, OP_RMOVE);
7257 ins->dreg = mono_alloc_freg (cfg);
7259 MONO_INST_NEW (cfg, ins, OP_MOVE);
7260 ins->dreg = mono_alloc_ireg (cfg);
7262 ins->sreg1 = in->dreg;
7265 case LLVMArgVtypeByVal:
7266 case LLVMArgVtypeByRef:
7267 case LLVMArgVtypeInReg:
7268 case LLVMArgVtypeAsScalar:
7269 case LLVMArgAsIArgs:
7270 case LLVMArgAsFpArgs:
7271 case LLVMArgGsharedvtVariable:
7272 case LLVMArgGsharedvtFixed:
7273 case LLVMArgGsharedvtFixedVtype:
7274 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
7275 ins->dreg = mono_alloc_ireg (cfg);
7276 ins->sreg1 = in->dreg;
7277 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
7278 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
7279 ins->inst_vtype = ainfo->type;
7280 ins->klass = mono_class_from_mono_type (ainfo->type);
7283 cfg->exception_message = g_strdup ("ainfo->storage");
7284 cfg->disable_llvm = TRUE;
7288 if (!cfg->disable_llvm) {
7289 MONO_ADD_INS (cfg->cbb, ins);
7290 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
7295 static unsigned char*
7296 alloc_cb (LLVMValueRef function, int size)
7300 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7304 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
7306 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
7311 emitted_cb (LLVMValueRef function, void *start, void *end)
7315 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7317 cfg->code_len = (guint8*)end - (guint8*)start;
7321 exception_cb (void *data)
7324 MonoJitExceptionInfo *ei;
7325 guint32 ei_len, i, j, nested_len, nindex;
7326 gpointer *type_info;
7327 int this_reg, this_offset;
7329 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7333 * data points to a DWARF FDE structure, convert it to our unwind format and
7335 * An alternative would be to save it directly, and modify our unwinder to work
7338 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
7339 if (cfg->verbose_level > 1)
7340 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
7342 /* Count nested clauses */
7344 for (i = 0; i < ei_len; ++i) {
7345 gint32 cindex1 = *(gint32*)type_info [i];
7346 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
7348 for (j = 0; j < cfg->header->num_clauses; ++j) {
7350 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
7352 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7358 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
7359 cfg->llvm_ex_info_len = ei_len + nested_len;
7360 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
7361 /* Fill the rest of the information from the type info */
7362 for (i = 0; i < ei_len; ++i) {
7363 gint32 clause_index = *(gint32*)type_info [i];
7364 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
7366 cfg->llvm_ex_info [i].flags = clause->flags;
7367 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
7368 cfg->llvm_ex_info [i].clause_index = clause_index;
7372 * For nested clauses, the LLVM produced exception info associates the try interval with
7373 * the innermost handler, while mono expects it to be associated with all nesting clauses.
7374 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
7375 * and everything else from the nested clause.
7378 for (i = 0; i < ei_len; ++i) {
7379 gint32 cindex1 = *(gint32*)type_info [i];
7380 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
7382 for (j = 0; j < cfg->header->num_clauses; ++j) {
7384 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
7385 MonoJitExceptionInfo *nesting_ei, *nested_ei;
7387 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7388 /* clause1 is the nested clause */
7389 nested_ei = &cfg->llvm_ex_info [i];
7390 nesting_ei = &cfg->llvm_ex_info [nindex];
7393 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
7395 nesting_ei->flags = clause2->flags;
7396 nesting_ei->data.catch_class = clause2->data.catch_class;
7397 nesting_ei->clause_index = cindex2;
7401 g_assert (nindex == ei_len + nested_len);
7402 cfg->llvm_this_reg = this_reg;
7403 cfg->llvm_this_offset = this_offset;
7405 /* type_info [i] is cfg mempool allocated, no need to free it */
7411 #if LLVM_API_VERSION > 100
7413 * decode_llvm_eh_info:
7415 * Decode the EH table emitted by llvm in jit mode, and store
7416 * the result into cfg.
7419 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
7421 MonoCompile *cfg = ctx->cfg;
7424 MonoLLVMFDEInfo info;
7425 MonoJitExceptionInfo *ei;
7426 guint8 *p = eh_frame;
7427 int version, fde_count, fde_offset;
7428 guint32 ei_len, i, nested_len;
7429 gpointer *type_info;
7433 * Decode the one element EH table emitted by the MonoException class
7437 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
7440 g_assert (version == 3);
7443 p = (guint8 *)ALIGN_PTR_TO (p, 4);
7445 fde_count = *(guint32*)p;
7449 g_assert (fde_count == 1);
7451 /* The only table entry */
7452 fde_offset = table [1];
7455 cfg->code_len = table [0];
7456 fde_len = table [1] - fde_offset;
7459 fde = (guint8*)eh_frame + fde_offset;
7460 cie = (guint8*)table;
7462 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info);
7464 cfg->encoded_unwind_ops = info.unw_info;
7465 cfg->encoded_unwind_ops_len = info.unw_info_len;
7466 if (cfg->verbose_level > 1)
7467 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
7468 if (info.this_reg != -1) {
7469 cfg->llvm_this_reg = info.this_reg;
7470 cfg->llvm_this_offset = info.this_offset;
7474 ei_len = info.ex_info_len;
7475 type_info = info.type_info;
7477 // Nested clauses are currently disabled
7480 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
7481 cfg->llvm_ex_info_len = ei_len + nested_len;
7482 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
7483 /* Fill the rest of the information from the type info */
7484 for (i = 0; i < ei_len; ++i) {
7485 gint32 clause_index = *(gint32*)type_info [i];
7486 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
7488 cfg->llvm_ex_info [i].flags = clause->flags;
7489 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
7490 cfg->llvm_ex_info [i].clause_index = clause_index;
7496 dlsym_cb (const char *name, void **symbol)
7502 if (!strcmp (name, "__bzero")) {
7503 *symbol = (void*)bzero;
7505 current = mono_dl_open (NULL, 0, NULL);
7508 err = mono_dl_symbol (current, name, symbol);
7510 mono_dl_close (current);
7512 #ifdef MONO_ARCH_HAVE_CREATE_LLVM_NATIVE_THUNK
7513 *symbol = (char*)mono_arch_create_llvm_native_thunk (mono_domain_get (), (guint8*)(*symbol));
7519 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
7521 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
7525 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
7527 LLVMTypeRef param_types [4];
7529 param_types [0] = param_type1;
7530 param_types [1] = param_type2;
7532 AddFunc (module, name, ret_type, param_types, 2);
7538 INTRINS_SADD_OVF_I32,
7539 INTRINS_UADD_OVF_I32,
7540 INTRINS_SSUB_OVF_I32,
7541 INTRINS_USUB_OVF_I32,
7542 INTRINS_SMUL_OVF_I32,
7543 INTRINS_UMUL_OVF_I32,
7544 INTRINS_SADD_OVF_I64,
7545 INTRINS_UADD_OVF_I64,
7546 INTRINS_SSUB_OVF_I64,
7547 INTRINS_USUB_OVF_I64,
7548 INTRINS_SMUL_OVF_I64,
7549 INTRINS_UMUL_OVF_I64,
7556 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7557 INTRINS_SSE_PMOVMSKB,
7558 INTRINS_SSE_PSRLI_W,
7559 INTRINS_SSE_PSRAI_W,
7560 INTRINS_SSE_PSLLI_W,
7561 INTRINS_SSE_PSRLI_D,
7562 INTRINS_SSE_PSRAI_D,
7563 INTRINS_SSE_PSLLI_D,
7564 INTRINS_SSE_PSRLI_Q,
7565 INTRINS_SSE_PSLLI_Q,
7566 INTRINS_SSE_SQRT_PD,
7567 INTRINS_SSE_SQRT_PS,
7568 INTRINS_SSE_RSQRT_PS,
7570 INTRINS_SSE_CVTTPD2DQ,
7571 INTRINS_SSE_CVTTPS2DQ,
7572 INTRINS_SSE_CVTDQ2PD,
7573 INTRINS_SSE_CVTDQ2PS,
7574 INTRINS_SSE_CVTPD2DQ,
7575 INTRINS_SSE_CVTPS2DQ,
7576 INTRINS_SSE_CVTPD2PS,
7577 INTRINS_SSE_CVTPS2PD,
7580 INTRINS_SSE_PACKSSWB,
7581 INTRINS_SSE_PACKUSWB,
7582 INTRINS_SSE_PACKSSDW,
7583 INTRINS_SSE_PACKUSDW,
7588 INTRINS_SSE_ADDSUBPS,
7593 INTRINS_SSE_ADDSUBPD,
7601 INTRINS_SSE_PADDUSW,
7602 INTRINS_SSE_PSUBUSW,
7610 INTRINS_SSE_PADDUSB,
7611 INTRINS_SSE_PSUBUSB,
7623 static IntrinsicDesc intrinsics[] = {
7624 {INTRINS_MEMSET, "llvm.memset.p0i8.i32"},
7625 {INTRINS_MEMCPY, "llvm.memcpy.p0i8.p0i8.i32"},
7626 {INTRINS_SADD_OVF_I32, "llvm.sadd.with.overflow.i32"},
7627 {INTRINS_UADD_OVF_I32, "llvm.uadd.with.overflow.i32"},
7628 {INTRINS_SSUB_OVF_I32, "llvm.ssub.with.overflow.i32"},
7629 {INTRINS_USUB_OVF_I32, "llvm.usub.with.overflow.i32"},
7630 {INTRINS_SMUL_OVF_I32, "llvm.smul.with.overflow.i32"},
7631 {INTRINS_UMUL_OVF_I32, "llvm.umul.with.overflow.i32"},
7632 {INTRINS_SADD_OVF_I64, "llvm.sadd.with.overflow.i64"},
7633 {INTRINS_UADD_OVF_I64, "llvm.uadd.with.overflow.i64"},
7634 {INTRINS_SSUB_OVF_I64, "llvm.ssub.with.overflow.i64"},
7635 {INTRINS_USUB_OVF_I64, "llvm.usub.with.overflow.i64"},
7636 {INTRINS_SMUL_OVF_I64, "llvm.smul.with.overflow.i64"},
7637 {INTRINS_UMUL_OVF_I64, "llvm.umul.with.overflow.i64"},
7638 {INTRINS_SIN, "llvm.sin.f64"},
7639 {INTRINS_COS, "llvm.cos.f64"},
7640 {INTRINS_SQRT, "llvm.sqrt.f64"},
7641 /* This isn't an intrinsic, instead llvm seems to special case it by name */
7642 {INTRINS_FABS, "fabs"},
7643 {INTRINS_EXPECT_I8, "llvm.expect.i8"},
7644 {INTRINS_EXPECT_I1, "llvm.expect.i1"},
7645 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7646 {INTRINS_SSE_PMOVMSKB, "llvm.x86.sse2.pmovmskb.128"},
7647 {INTRINS_SSE_PSRLI_W, "llvm.x86.sse2.psrli.w"},
7648 {INTRINS_SSE_PSRAI_W, "llvm.x86.sse2.psrai.w"},
7649 {INTRINS_SSE_PSLLI_W, "llvm.x86.sse2.pslli.w"},
7650 {INTRINS_SSE_PSRLI_D, "llvm.x86.sse2.psrli.d"},
7651 {INTRINS_SSE_PSRAI_D, "llvm.x86.sse2.psrai.d"},
7652 {INTRINS_SSE_PSLLI_D, "llvm.x86.sse2.pslli.d"},
7653 {INTRINS_SSE_PSRLI_Q, "llvm.x86.sse2.psrli.q"},
7654 {INTRINS_SSE_PSLLI_Q, "llvm.x86.sse2.pslli.q"},
7655 {INTRINS_SSE_SQRT_PD, "llvm.x86.sse2.sqrt.pd"},
7656 {INTRINS_SSE_SQRT_PS, "llvm.x86.sse.sqrt.ps"},
7657 {INTRINS_SSE_RSQRT_PS, "llvm.x86.sse.rsqrt.ps"},
7658 {INTRINS_SSE_RCP_PS, "llvm.x86.sse.rcp.ps"},
7659 {INTRINS_SSE_CVTTPD2DQ, "llvm.x86.sse2.cvttpd2dq"},
7660 {INTRINS_SSE_CVTTPS2DQ, "llvm.x86.sse2.cvttps2dq"},
7661 {INTRINS_SSE_CVTDQ2PD, "llvm.x86.sse2.cvtdq2pd"},
7662 {INTRINS_SSE_CVTDQ2PS, "llvm.x86.sse2.cvtdq2ps"},
7663 {INTRINS_SSE_CVTPD2DQ, "llvm.x86.sse2.cvtpd2dq"},
7664 {INTRINS_SSE_CVTPS2DQ, "llvm.x86.sse2.cvtps2dq"},
7665 {INTRINS_SSE_CVTPD2PS, "llvm.x86.sse2.cvtpd2ps"},
7666 {INTRINS_SSE_CVTPS2PD, "llvm.x86.sse2.cvtps2pd"},
7667 {INTRINS_SSE_CMPPD, "llvm.x86.sse2.cmp.pd"},
7668 {INTRINS_SSE_CMPPS, "llvm.x86.sse.cmp.ps"},
7669 {INTRINS_SSE_PACKSSWB, "llvm.x86.sse2.packsswb.128"},
7670 {INTRINS_SSE_PACKUSWB, "llvm.x86.sse2.packuswb.128"},
7671 {INTRINS_SSE_PACKSSDW, "llvm.x86.sse2.packssdw.128"},
7672 {INTRINS_SSE_PACKUSDW, "llvm.x86.sse41.packusdw"},
7673 {INTRINS_SSE_MINPS, "llvm.x86.sse.min.ps"},
7674 {INTRINS_SSE_MAXPS, "llvm.x86.sse.max.ps"},
7675 {INTRINS_SSE_HADDPS, "llvm.x86.sse3.hadd.ps"},
7676 {INTRINS_SSE_HSUBPS, "llvm.x86.sse3.hsub.ps"},
7677 {INTRINS_SSE_ADDSUBPS, "llvm.x86.sse3.addsub.ps"},
7678 {INTRINS_SSE_MINPD, "llvm.x86.sse2.min.pd"},
7679 {INTRINS_SSE_MAXPD, "llvm.x86.sse2.max.pd"},
7680 {INTRINS_SSE_HADDPD, "llvm.x86.sse3.hadd.pd"},
7681 {INTRINS_SSE_HSUBPD, "llvm.x86.sse3.hsub.pd"},
7682 {INTRINS_SSE_ADDSUBPD, "llvm.x86.sse3.addsub.pd"},
7683 {INTRINS_SSE_PMINUD, "llvm.x86.sse41.pminud"},
7684 {INTRINS_SSE_PMAXUD, "llvm.x86.sse41.pmaxud"},
7685 {INTRINS_SSE_PMINUW, "llvm.x86.sse41.pminuw"},
7686 {INTRINS_SSE_PMINSW, "llvm.x86.sse2.pmins.w"},
7687 {INTRINS_SSE_PMAXUW, "llvm.x86.sse41.pmaxuw"},
7688 {INTRINS_SSE_PADDSW, "llvm.x86.sse2.padds.w"},
7689 {INTRINS_SSE_PSUBSW, "llvm.x86.sse2.psubs.w"},
7690 {INTRINS_SSE_PADDUSW, "llvm.x86.sse2.paddus.w"},
7691 {INTRINS_SSE_PSUBUSW, "llvm.x86.sse2.psubus.w"},
7692 {INTRINS_SSE_PAVGW, "llvm.x86.sse2.pavg.w"},
7693 {INTRINS_SSE_PMULHW, "llvm.x86.sse2.pmulh.w"},
7694 {INTRINS_SSE_PMULHU, "llvm.x86.sse2.pmulhu.w"},
7695 {INTRINS_SSE_PMINUB, "llvm.x86.sse2.pminu.b"},
7696 {INTRINS_SSE_PMAXUB, "llvm.x86.sse2.pmaxu.b"},
7697 {INTRINS_SE_PADDSB, "llvm.x86.sse2.padds.b"},
7698 {INTRINS_SSE_PSUBSB, "llvm.x86.sse2.psubs.b"},
7699 {INTRINS_SSE_PADDUSB, "llvm.x86.sse2.paddus.b"},
7700 {INTRINS_SSE_PSUBUSB, "llvm.x86.sse2.psubus.b"},
7701 {INTRINS_SSE_PAVGB, "llvm.x86.sse2.pavg.b"},
7702 {INTRINS_SSE_PAUSE, "llvm.x86.sse2.pause"}
7707 add_sse_binary (LLVMModuleRef module, const char *name, int type)
7709 LLVMTypeRef ret_type = type_to_simd_type (type);
7710 AddFunc2 (module, name, ret_type, ret_type, ret_type);
7714 add_intrinsic (LLVMModuleRef module, int id)
7717 LLVMTypeRef ret_type, arg_types [16];
7719 name = g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
7723 case INTRINS_MEMSET: {
7724 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
7726 AddFunc (module, name, LLVMVoidType (), params, 5);
7729 case INTRINS_MEMCPY: {
7730 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
7732 AddFunc (module, name, LLVMVoidType (), params, 5);
7735 case INTRINS_SADD_OVF_I32:
7736 case INTRINS_UADD_OVF_I32:
7737 case INTRINS_SSUB_OVF_I32:
7738 case INTRINS_USUB_OVF_I32:
7739 case INTRINS_SMUL_OVF_I32:
7740 case INTRINS_UMUL_OVF_I32: {
7741 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
7742 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
7743 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
7745 AddFunc (module, name, ret_type, params, 2);
7748 case INTRINS_SADD_OVF_I64:
7749 case INTRINS_UADD_OVF_I64:
7750 case INTRINS_SSUB_OVF_I64:
7751 case INTRINS_USUB_OVF_I64:
7752 case INTRINS_SMUL_OVF_I64:
7753 case INTRINS_UMUL_OVF_I64: {
7754 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
7755 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
7756 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
7758 AddFunc (module, name, ret_type, params, 2);
7764 case INTRINS_FABS: {
7765 LLVMTypeRef params [] = { LLVMDoubleType () };
7767 AddFunc (module, name, LLVMDoubleType (), params, 1);
7770 case INTRINS_EXPECT_I8:
7771 AddFunc2 (module, name, LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
7773 case INTRINS_EXPECT_I1:
7774 AddFunc2 (module, name, LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
7776 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7777 case INTRINS_SSE_PMOVMSKB:
7779 ret_type = LLVMInt32Type ();
7780 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
7781 AddFunc (module, name, ret_type, arg_types, 1);
7783 case INTRINS_SSE_PSRLI_W:
7784 case INTRINS_SSE_PSRAI_W:
7785 case INTRINS_SSE_PSLLI_W:
7787 ret_type = type_to_simd_type (MONO_TYPE_I2);
7788 arg_types [0] = ret_type;
7789 arg_types [1] = LLVMInt32Type ();
7790 AddFunc (module, name, ret_type, arg_types, 2);
7792 case INTRINS_SSE_PSRLI_D:
7793 case INTRINS_SSE_PSRAI_D:
7794 case INTRINS_SSE_PSLLI_D:
7795 ret_type = type_to_simd_type (MONO_TYPE_I4);
7796 arg_types [0] = ret_type;
7797 arg_types [1] = LLVMInt32Type ();
7798 AddFunc (module, name, ret_type, arg_types, 2);
7800 case INTRINS_SSE_PSRLI_Q:
7801 case INTRINS_SSE_PSLLI_Q:
7802 ret_type = type_to_simd_type (MONO_TYPE_I8);
7803 arg_types [0] = ret_type;
7804 arg_types [1] = LLVMInt32Type ();
7805 AddFunc (module, name, ret_type, arg_types, 2);
7807 case INTRINS_SSE_SQRT_PD:
7809 ret_type = type_to_simd_type (MONO_TYPE_R8);
7810 arg_types [0] = ret_type;
7811 AddFunc (module, name, ret_type, arg_types, 1);
7813 case INTRINS_SSE_SQRT_PS:
7814 ret_type = type_to_simd_type (MONO_TYPE_R4);
7815 arg_types [0] = ret_type;
7816 AddFunc (module, name, ret_type, arg_types, 1);
7818 case INTRINS_SSE_RSQRT_PS:
7819 ret_type = type_to_simd_type (MONO_TYPE_R4);
7820 arg_types [0] = ret_type;
7821 AddFunc (module, name, ret_type, arg_types, 1);
7823 case INTRINS_SSE_RCP_PS:
7824 ret_type = type_to_simd_type (MONO_TYPE_R4);
7825 arg_types [0] = ret_type;
7826 AddFunc (module, name, ret_type, arg_types, 1);
7828 case INTRINS_SSE_CVTTPD2DQ:
7829 ret_type = type_to_simd_type (MONO_TYPE_I4);
7830 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7831 AddFunc (module, name, ret_type, arg_types, 1);
7833 case INTRINS_SSE_CVTTPS2DQ:
7834 ret_type = type_to_simd_type (MONO_TYPE_I4);
7835 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7836 AddFunc (module, name, ret_type, arg_types, 1);
7838 case INTRINS_SSE_CVTDQ2PD:
7839 /* Conversion ops */
7840 ret_type = type_to_simd_type (MONO_TYPE_R8);
7841 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
7842 AddFunc (module, name, ret_type, arg_types, 1);
7844 case INTRINS_SSE_CVTDQ2PS:
7845 ret_type = type_to_simd_type (MONO_TYPE_R4);
7846 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
7847 AddFunc (module, name, ret_type, arg_types, 1);
7849 case INTRINS_SSE_CVTPD2DQ:
7850 ret_type = type_to_simd_type (MONO_TYPE_I4);
7851 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7852 AddFunc (module, name, ret_type, arg_types, 1);
7854 case INTRINS_SSE_CVTPS2DQ:
7855 ret_type = type_to_simd_type (MONO_TYPE_I4);
7856 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7857 AddFunc (module, name, ret_type, arg_types, 1);
7859 case INTRINS_SSE_CVTPD2PS:
7860 ret_type = type_to_simd_type (MONO_TYPE_R4);
7861 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7862 AddFunc (module, name, ret_type, arg_types, 1);
7864 case INTRINS_SSE_CVTPS2PD:
7865 ret_type = type_to_simd_type (MONO_TYPE_R8);
7866 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7867 AddFunc (module, name, ret_type, arg_types, 1);
7869 case INTRINS_SSE_CMPPD:
7871 ret_type = type_to_simd_type (MONO_TYPE_R8);
7872 arg_types [0] = ret_type;
7873 arg_types [1] = ret_type;
7874 arg_types [2] = LLVMInt8Type ();
7875 AddFunc (module, name, ret_type, arg_types, 3);
7877 case INTRINS_SSE_CMPPS:
7878 ret_type = type_to_simd_type (MONO_TYPE_R4);
7879 arg_types [0] = ret_type;
7880 arg_types [1] = ret_type;
7881 arg_types [2] = LLVMInt8Type ();
7882 AddFunc (module, name, ret_type, arg_types, 3);
7884 case INTRINS_SSE_PACKSSWB:
7885 case INTRINS_SSE_PACKUSWB:
7886 case INTRINS_SSE_PACKSSDW:
7888 ret_type = type_to_simd_type (MONO_TYPE_I1);
7889 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
7890 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
7891 AddFunc (module, name, ret_type, arg_types, 2);
7893 case INTRINS_SSE_PACKUSDW:
7894 ret_type = type_to_simd_type (MONO_TYPE_I2);
7895 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
7896 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
7897 AddFunc (module, name, ret_type, arg_types, 2);
7899 /* SSE Binary ops */
7900 case INTRINS_SSE_PMINUD:
7901 case INTRINS_SSE_PMAXUD:
7902 add_sse_binary (module, name, MONO_TYPE_I4);
7904 case INTRINS_SSE_PMINUW:
7905 case INTRINS_SSE_PMINSW:
7906 case INTRINS_SSE_PMAXUW:
7907 case INTRINS_SSE_PADDSW:
7908 case INTRINS_SSE_PSUBSW:
7909 case INTRINS_SSE_PADDUSW:
7910 case INTRINS_SSE_PSUBUSW:
7911 case INTRINS_SSE_PAVGW:
7912 case INTRINS_SSE_PMULHW:
7913 case INTRINS_SSE_PMULHU:
7914 add_sse_binary (module, name, MONO_TYPE_I2);
7916 case INTRINS_SSE_MINPS:
7917 case INTRINS_SSE_MAXPS:
7918 case INTRINS_SSE_HADDPS:
7919 case INTRINS_SSE_HSUBPS:
7920 case INTRINS_SSE_ADDSUBPS:
7921 add_sse_binary (module, name, MONO_TYPE_R4);
7923 case INTRINS_SSE_MINPD:
7924 case INTRINS_SSE_MAXPD:
7925 case INTRINS_SSE_HADDPD:
7926 case INTRINS_SSE_HSUBPD:
7927 case INTRINS_SSE_ADDSUBPD:
7928 add_sse_binary (module, name, MONO_TYPE_R8);
7930 case INTRINS_SSE_PMINUB:
7931 case INTRINS_SSE_PMAXUB:
7932 case INTRINS_SE_PADDSB:
7933 case INTRINS_SSE_PSUBSB:
7934 case INTRINS_SSE_PADDUSB:
7935 case INTRINS_SSE_PSUBUSB:
7936 case INTRINS_SSE_PAVGB:
7937 add_sse_binary (module, name, MONO_TYPE_I1);
7939 case INTRINS_SSE_PAUSE:
7940 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
7944 g_assert_not_reached ();
7950 get_intrinsic (EmitContext *ctx, const char *name)
7952 #if LLVM_API_VERSION > 100
7956 * Every method is emitted into its own module so
7957 * we can add intrinsics on demand.
7959 res = LLVMGetNamedFunction (ctx->lmodule, name);
7963 /* No locking needed */
7964 id = GPOINTER_TO_INT (g_hash_table_lookup (intrins_name_to_id, name));
7967 printf ("%s\n", name);
7968 g_assert (id != -1);
7969 add_intrinsic (ctx->lmodule, id);
7970 res = LLVMGetNamedFunction (ctx->lmodule, name);
7978 res = LLVMGetNamedFunction (ctx->lmodule, name);
7985 add_intrinsics (LLVMModuleRef module)
7989 /* Emit declarations of instrinsics */
7991 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
7992 * type doesn't seem to do any locking.
7994 for (i = 0; i < INTRINS_NUM; ++i)
7995 add_intrinsic (module, i);
7999 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
8001 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
8004 /* SSE intrinsics */
8005 #if defined(TARGET_X86) || defined(TARGET_AMD64)
8009 /* Load/Store intrinsics */
8011 LLVMTypeRef arg_types [5];
8015 for (i = 1; i <= 8; i *= 2) {
8016 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
8017 arg_types [1] = LLVMInt32Type ();
8018 arg_types [2] = LLVMInt1Type ();
8019 arg_types [3] = LLVMInt32Type ();
8020 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
8021 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
8023 arg_types [0] = LLVMIntType (i * 8);
8024 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
8025 arg_types [2] = LLVMInt32Type ();
8026 arg_types [3] = LLVMInt1Type ();
8027 arg_types [4] = LLVMInt32Type ();
8028 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
8029 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
8035 add_types (MonoLLVMModule *module)
8037 module->ptr_type = LLVMPointerType (sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
8041 mono_llvm_init (void)
8046 mono_native_tls_alloc (¤t_cfg_tls_id, NULL);
8048 h = g_hash_table_new (NULL, NULL);
8049 for (i = 0; i < INTRINS_NUM; ++i)
8050 g_hash_table_insert (h, GINT_TO_POINTER (intrinsics [i].id), (gpointer)intrinsics [i].name);
8051 intrins_id_to_name = h;
8053 h = g_hash_table_new (g_str_hash, g_str_equal);
8054 for (i = 0; i < INTRINS_NUM; ++i)
8055 g_hash_table_insert (h, (gpointer)intrinsics [i].name, GINT_TO_POINTER (intrinsics [i].id + 1));
8056 intrins_name_to_id = h;
8060 init_jit_module (MonoDomain *domain)
8062 MonoJitDomainInfo *dinfo;
8063 MonoLLVMModule *module;
8066 dinfo = domain_jit_info (domain);
8067 if (dinfo->llvm_module)
8070 mono_loader_lock ();
8072 if (dinfo->llvm_module) {
8073 mono_loader_unlock ();
8077 module = g_new0 (MonoLLVMModule, 1);
8079 name = g_strdup_printf ("mono-%s", domain->friendly_name);
8080 module->lmodule = LLVMModuleCreateWithName (name);
8081 module->context = LLVMGetGlobalContext ();
8083 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->lmodule), alloc_cb, emitted_cb, exception_cb, dlsym_cb, &module->ee);
8085 add_intrinsics (module->lmodule);
8088 module->llvm_types = g_hash_table_new (NULL, NULL);
8090 #if LLVM_API_VERSION < 100
8091 MonoJitICallInfo *info;
8093 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
8095 LLVMAddGlobalMapping (module->ee, LLVMGetNamedFunction (module->lmodule, "llvm_resume_unwind_trampoline"), (void*)info->func);
8098 mono_memory_barrier ();
8100 dinfo->llvm_module = module;
8102 mono_loader_unlock ();
8106 mono_llvm_cleanup (void)
8108 MonoLLVMModule *module = &aot_module;
8110 if (module->lmodule)
8111 LLVMDisposeModule (module->lmodule);
8113 if (module->context)
8114 LLVMContextDispose (module->context);
8118 mono_llvm_free_domain_info (MonoDomain *domain)
8120 MonoJitDomainInfo *info = domain_jit_info (domain);
8121 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
8127 if (module->llvm_types)
8128 g_hash_table_destroy (module->llvm_types);
8130 mono_llvm_dispose_ee (module->mono_ee);
8132 if (module->bb_names) {
8133 for (i = 0; i < module->bb_names_len; ++i)
8134 g_free (module->bb_names [i]);
8135 g_free (module->bb_names);
8137 //LLVMDisposeModule (module->module);
8141 info->llvm_module = NULL;
8145 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, gboolean emit_dwarf, gboolean static_link, gboolean llvm_only)
8147 MonoLLVMModule *module = &aot_module;
8149 /* Delete previous module */
8150 if (module->plt_entries)
8151 g_hash_table_destroy (module->plt_entries);
8152 if (module->lmodule)
8153 LLVMDisposeModule (module->lmodule);
8155 memset (module, 0, sizeof (aot_module));
8157 module->lmodule = LLVMModuleCreateWithName ("aot");
8158 module->assembly = assembly;
8159 module->global_prefix = g_strdup (global_prefix);
8160 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
8161 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
8162 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
8163 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
8164 module->external_symbols = TRUE;
8165 module->emit_dwarf = emit_dwarf;
8166 module->static_link = static_link;
8167 module->llvm_only = llvm_only;
8168 /* The first few entries are reserved */
8169 module->max_got_offset = 16;
8170 module->context = LLVMContextCreate ();
8173 /* clang ignores our debug info because it has an invalid version */
8174 module->emit_dwarf = FALSE;
8176 #if LLVM_API_VERSION > 100
8177 module->emit_dwarf = FALSE;
8180 add_intrinsics (module->lmodule);
8183 #if LLVM_API_VERSION > 100
8184 if (module->emit_dwarf) {
8185 char *dir, *build_info, *s, *cu_name;
8187 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
8190 dir = g_strdup (".");
8191 build_info = mono_get_runtime_build_info ();
8192 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
8193 cu_name = g_path_get_basename (assembly->image->name);
8194 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
8196 g_free (build_info);
8203 * We couldn't compute the type of the LLVM global representing the got because
8204 * its size is only known after all the methods have been emitted. So create
8205 * a dummy variable, and replace all uses it with the real got variable when
8206 * its size is known in mono_llvm_emit_aot_module ().
8209 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
8211 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
8212 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
8215 /* Add initialization array */
8217 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
8219 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
8220 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
8224 emit_init_icall_wrappers (module);
8226 emit_llvm_code_start (module);
8228 /* Add a dummy personality function */
8229 if (!use_debug_personality) {
8230 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
8231 LLVMSetLinkage (personality, LLVMExternalLinkage);
8232 mark_as_used (module, personality);
8235 /* Add a reference to the c++ exception we throw/catch */
8237 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
8238 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
8239 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
8240 mono_llvm_set_is_constant (module->sentinel_exception);
8243 module->llvm_types = g_hash_table_new (NULL, NULL);
8244 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
8245 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
8246 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
8247 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
8248 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
8249 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
8250 module->method_to_callers = g_hash_table_new (NULL, NULL);
8254 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
8257 LLVMValueRef res, *vals;
8259 vals = g_new0 (LLVMValueRef, nvalues);
8260 for (i = 0; i < nvalues; ++i)
8261 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
8262 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
8268 * mono_llvm_emit_aot_file_info:
8270 * Emit the MonoAotFileInfo structure.
8271 * Same as emit_aot_file_info () in aot-compiler.c.
8274 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
8276 MonoLLVMModule *module = &aot_module;
8278 /* Save these for later */
8279 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
8280 module->has_jitted_code = has_jitted_code;
8284 * mono_llvm_emit_aot_data:
8286 * Emit the binary data DATA pointed to by symbol SYMBOL.
8289 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
8291 MonoLLVMModule *module = &aot_module;
8295 type = LLVMArrayType (LLVMInt8Type (), data_len);
8296 d = LLVMAddGlobal (module->lmodule, type, symbol);
8297 LLVMSetVisibility (d, LLVMHiddenVisibility);
8298 LLVMSetLinkage (d, LLVMInternalLinkage);
8299 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
8300 mono_llvm_set_is_constant (d);
8303 /* Add a reference to a global defined in JITted code */
8305 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
8310 s = g_strdup_printf ("%s%s", module->global_prefix, name);
8311 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
8317 emit_aot_file_info (MonoLLVMModule *module)
8319 LLVMTypeRef file_info_type;
8320 LLVMTypeRef *eltypes, eltype;
8321 LLVMValueRef info_var;
8322 LLVMValueRef *fields;
8323 int i, nfields, tindex;
8324 MonoAotFileInfo *info;
8325 LLVMModuleRef lmodule = module->lmodule;
8327 info = &module->aot_info;
8329 /* Create an LLVM type to represent MonoAotFileInfo */
8330 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 15 + 5;
8331 eltypes = g_new (LLVMTypeRef, nfields);
8333 eltypes [tindex ++] = LLVMInt32Type ();
8334 eltypes [tindex ++] = LLVMInt32Type ();
8336 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
8337 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
8339 for (i = 0; i < 15; ++i)
8340 eltypes [tindex ++] = LLVMInt32Type ();
8342 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
8343 for (i = 0; i < 4; ++i)
8344 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
8345 g_assert (tindex == nfields);
8346 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
8347 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
8349 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
8350 if (module->static_link) {
8351 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
8352 LLVMSetLinkage (info_var, LLVMInternalLinkage);
8354 fields = g_new (LLVMValueRef, nfields);
8356 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
8357 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
8361 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
8362 * for symbols defined in the .s file emitted by the aot compiler.
8364 eltype = eltypes [tindex];
8365 if (module->llvm_only)
8366 fields [tindex ++] = LLVMConstNull (eltype);
8368 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
8369 fields [tindex ++] = module->got_var;
8370 /* llc defines this directly */
8371 if (!module->llvm_only) {
8372 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
8373 fields [tindex ++] = LLVMConstNull (eltype);
8374 fields [tindex ++] = LLVMConstNull (eltype);
8376 fields [tindex ++] = LLVMConstNull (eltype);
8377 fields [tindex ++] = module->get_method;
8378 fields [tindex ++] = module->get_unbox_tramp;
8380 if (module->has_jitted_code) {
8381 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
8382 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
8384 fields [tindex ++] = LLVMConstNull (eltype);
8385 fields [tindex ++] = LLVMConstNull (eltype);
8387 if (!module->llvm_only)
8388 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
8390 fields [tindex ++] = LLVMConstNull (eltype);
8391 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
8392 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
8393 fields [tindex ++] = LLVMConstNull (eltype);
8395 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
8396 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
8397 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
8398 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
8399 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
8400 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
8401 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
8402 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
8403 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
8404 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
8406 /* Not needed (mem_end) */
8407 fields [tindex ++] = LLVMConstNull (eltype);
8408 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
8409 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
8410 if (info->trampoline_size [0]) {
8411 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
8412 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
8413 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_thunks");
8414 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
8416 fields [tindex ++] = LLVMConstNull (eltype);
8417 fields [tindex ++] = LLVMConstNull (eltype);
8418 fields [tindex ++] = LLVMConstNull (eltype);
8419 fields [tindex ++] = LLVMConstNull (eltype);
8421 if (module->static_link && !module->llvm_only)
8422 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
8424 fields [tindex ++] = LLVMConstNull (eltype);
8425 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
8426 if (!module->llvm_only) {
8427 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
8428 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
8429 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
8430 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
8431 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
8432 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
8434 fields [tindex ++] = LLVMConstNull (eltype);
8435 fields [tindex ++] = LLVMConstNull (eltype);
8436 fields [tindex ++] = LLVMConstNull (eltype);
8437 fields [tindex ++] = LLVMConstNull (eltype);
8438 fields [tindex ++] = LLVMConstNull (eltype);
8439 fields [tindex ++] = LLVMConstNull (eltype);
8442 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
8443 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
8446 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
8447 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
8448 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
8449 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
8450 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
8451 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
8452 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
8453 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
8454 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
8455 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
8456 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
8457 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
8458 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
8459 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
8460 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
8462 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
8463 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
8464 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
8465 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
8466 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
8467 g_assert (tindex == nfields);
8469 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
8471 if (module->static_link) {
8475 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
8476 /* Get rid of characters which cannot occur in symbols */
8478 for (p = s; *p; ++p) {
8479 if (!(isalnum (*p) || *p == '_'))
8482 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
8484 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
8485 LLVMSetLinkage (var, LLVMExternalLinkage);
8490 * Emit the aot module into the LLVM bitcode file FILENAME.
8493 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
8495 LLVMTypeRef got_type, inited_type;
8496 LLVMValueRef real_got, real_inited;
8497 MonoLLVMModule *module = &aot_module;
8499 emit_llvm_code_end (module);
8502 * Create the real got variable and replace all uses of the dummy variable with
8505 got_type = LLVMArrayType (module->ptr_type, module->max_got_offset + 1);
8506 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
8507 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
8508 if (module->external_symbols) {
8509 LLVMSetLinkage (real_got, LLVMExternalLinkage);
8510 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
8512 LLVMSetLinkage (real_got, LLVMInternalLinkage);
8514 mono_llvm_replace_uses_of (module->got_var, real_got);
8516 mark_as_used (&aot_module, real_got);
8518 /* Delete the dummy got so it doesn't become a global */
8519 LLVMDeleteGlobal (module->got_var);
8520 module->got_var = real_got;
8523 * Same for the init_var
8525 if (module->llvm_only) {
8526 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
8527 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
8528 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
8529 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
8530 mono_llvm_replace_uses_of (module->inited_var, real_inited);
8531 LLVMDeleteGlobal (module->inited_var);
8534 if (module->llvm_only) {
8535 emit_get_method (&aot_module);
8536 emit_get_unbox_tramp (&aot_module);
8539 emit_llvm_used (&aot_module);
8540 emit_dbg_info (&aot_module, filename, cu_name);
8541 emit_aot_file_info (&aot_module);
8544 * Replace GOT entries for directly callable methods with the methods themselves.
8545 * It would be easier to implement this by predefining all methods before compiling
8546 * their bodies, but that couldn't handle the case when a method fails to compile
8549 if (module->llvm_only) {
8550 GHashTableIter iter;
8552 GSList *callers, *l;
8554 g_hash_table_iter_init (&iter, module->method_to_callers);
8555 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
8556 LLVMValueRef lmethod;
8558 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
8560 for (l = callers; l; l = l->next) {
8561 LLVMValueRef caller = (LLVMValueRef)l->data;
8563 mono_llvm_replace_uses_of (caller, lmethod);
8569 /* Replace PLT entries for directly callable methods with the methods themselves */
8571 GHashTableIter iter;
8573 LLVMValueRef callee;
8575 g_hash_table_iter_init (&iter, module->plt_entries_ji);
8576 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
8577 if (mono_aot_is_direct_callable (ji)) {
8578 LLVMValueRef lmethod;
8580 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
8581 /* The types might not match because the caller might pass an rgctx */
8582 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
8583 mono_llvm_replace_uses_of (callee, lmethod);
8584 mono_aot_mark_unused_llvm_plt_entry (ji);
8594 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
8595 g_assert_not_reached ();
8600 LLVMWriteBitcodeToFile (module->lmodule, filename);
8605 md_string (const char *s)
8607 return LLVMMDString (s, strlen (s));
8610 /* Debugging support */
8613 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
8615 LLVMModuleRef lmodule = module->lmodule;
8616 LLVMValueRef args [16], ver;
8619 * This can only be enabled when LLVM code is emitted into a separate object
8620 * file, since the AOT compiler also emits dwarf info,
8621 * and the abbrev indexes will not be correct since llvm has added its own
8624 if (!module->emit_dwarf)
8627 #if LLVM_API_VERSION > 100
8628 mono_llvm_di_builder_finalize (module->di_builder);
8630 LLVMValueRef cu_args [16], cu;
8632 char *build_info, *s, *dir;
8635 * Emit dwarf info in the form of LLVM metadata. There is some
8636 * out-of-date documentation at:
8637 * http://llvm.org/docs/SourceLevelDebugging.html
8638 * but most of this was gathered from the llvm and
8643 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_TAG_compile_unit, FALSE);
8644 /* CU name/compilation dir */
8645 dir = g_path_get_dirname (filename);
8646 args [0] = LLVMMDString (cu_name, strlen (cu_name));
8647 args [1] = LLVMMDString (dir, strlen (dir));
8648 cu_args [n_cuargs ++] = LLVMMDNode (args, 2);
8651 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_LANG_C99, FALSE);
8653 build_info = mono_get_runtime_build_info ();
8654 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
8655 cu_args [n_cuargs ++] = LLVMMDString (s, strlen (s));
8656 g_free (build_info);
8658 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8660 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
8661 /* Runtime version */
8662 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8664 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8665 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8667 if (module->subprogram_mds) {
8671 mds = g_new0 (LLVMValueRef, module->subprogram_mds->len);
8672 for (i = 0; i < module->subprogram_mds->len; ++i)
8673 mds [i] = (LLVMValueRef)g_ptr_array_index (module->subprogram_mds, i);
8674 cu_args [n_cuargs ++] = LLVMMDNode (mds, module->subprogram_mds->len);
8676 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8679 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8680 /* Imported modules */
8681 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8683 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
8684 /* DebugEmissionKind = FullDebug */
8685 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8686 cu = LLVMMDNode (cu_args, n_cuargs);
8687 LLVMAddNamedMetadataOperand (lmodule, "llvm.dbg.cu", cu);
8690 #if LLVM_API_VERSION > 100
8691 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8692 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
8693 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8694 ver = LLVMMDNode (args, 3);
8695 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8697 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8698 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
8699 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
8700 ver = LLVMMDNode (args, 3);
8701 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8703 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8704 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
8705 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8706 ver = LLVMMDNode (args, 3);
8707 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8709 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8710 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
8711 args [2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8712 ver = LLVMMDNode (args, 3);
8713 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8718 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
8720 MonoLLVMModule *module = ctx->module;
8721 MonoDebugMethodInfo *minfo = ctx->minfo;
8722 char *source_file, *dir, *filename;
8723 LLVMValueRef md, args [16], ctx_args [16], md_args [64], type_args [16], ctx_md, type_md;
8724 MonoSymSeqPoint *sym_seq_points;
8730 mono_debug_symfile_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
8732 source_file = g_strdup ("<unknown>");
8733 dir = g_path_get_dirname (source_file);
8734 filename = g_path_get_basename (source_file);
8736 #if LLVM_API_VERSION > 100
8737 return mono_llvm_di_create_function (module->di_builder, module->cu, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
8740 ctx_args [0] = LLVMConstInt (LLVMInt32Type (), 0x29, FALSE);
8741 args [0] = md_string (filename);
8742 args [1] = md_string (dir);
8743 ctx_args [1] = LLVMMDNode (args, 2);
8744 ctx_md = LLVMMDNode (ctx_args, 2);
8746 type_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subroutine_type, FALSE);
8747 type_args [1] = NULL;
8748 type_args [2] = NULL;
8749 type_args [3] = LLVMMDString ("", 0);
8750 type_args [4] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8751 type_args [5] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8752 type_args [6] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8753 type_args [7] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8754 type_args [8] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8755 type_args [9] = NULL;
8756 type_args [10] = NULL;
8757 type_args [11] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8758 type_args [12] = NULL;
8759 type_args [13] = NULL;
8760 type_args [14] = NULL;
8761 type_md = LLVMMDNode (type_args, 14);
8763 /* http://llvm.org/docs/SourceLevelDebugging.html#subprogram-descriptors */
8764 md_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subprogram, FALSE);
8765 /* Source directory + file pair */
8766 args [0] = md_string (filename);
8767 args [1] = md_string (dir);
8768 md_args [1] = LLVMMDNode (args ,2);
8769 md_args [2] = ctx_md;
8770 md_args [3] = md_string (cfg->method->name);
8771 md_args [4] = md_string (name);
8772 md_args [5] = md_string (name);
8775 md_args [6] = LLVMConstInt (LLVMInt32Type (), sym_seq_points [0].line, FALSE);
8777 md_args [6] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8779 md_args [7] = type_md;
8781 md_args [8] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8783 md_args [9] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8785 md_args [10] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8786 /* Index into a virtual function */
8787 md_args [11] = NULL;
8788 md_args [12] = NULL;
8790 md_args [13] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8792 md_args [14] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8793 /* Pointer to LLVM function */
8794 md_args [15] = method;
8795 /* Function template parameter */
8796 md_args [16] = NULL;
8797 /* Function declaration descriptor */
8798 md_args [17] = NULL;
8799 /* List of function variables */
8800 md_args [18] = LLVMMDNode (args, 0);
8802 md_args [19] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8803 md = LLVMMDNode (md_args, 20);
8805 if (!module->subprogram_mds)
8806 module->subprogram_mds = g_ptr_array_new ();
8807 g_ptr_array_add (module->subprogram_mds, md);
8811 g_free (source_file);
8812 g_free (sym_seq_points);
8818 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
8820 MonoCompile *cfg = ctx->cfg;
8822 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
8823 MonoDebugSourceLocation *loc;
8824 LLVMValueRef loc_md;
8826 loc = mono_debug_symfile_lookup_location (ctx->minfo, cil_code - cfg->header->code);
8829 #if LLVM_API_VERSION > 100
8830 loc_md = mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
8831 mono_llvm_di_set_location (builder, loc_md);
8833 LLVMValueRef md_args [16];
8837 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->row, FALSE);
8838 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->column, FALSE);
8839 md_args [nmd_args ++] = ctx->dbg_md;
8840 md_args [nmd_args ++] = NULL;
8841 loc_md = LLVMMDNode (md_args, nmd_args);
8842 LLVMSetCurrentDebugLocation (builder, loc_md);
8844 mono_debug_symfile_free_location (loc);
8850 default_mono_llvm_unhandled_exception (void)
8852 MonoJitTlsData *jit_tls = mono_get_jit_tls ();
8853 MonoObject *target = mono_gchandle_get_target (jit_tls->thrown_exc);
8855 mono_unhandled_exception (target);
8856 exit (mono_environment_exitcode_get ());
8861 - Emit LLVM IR from the mono IR using the LLVM C API.
8862 - The original arch specific code remains, so we can fall back to it if we run
8863 into something we can't handle.
8867 A partial list of issues:
8868 - Handling of opcodes which can throw exceptions.
8870 In the mono JIT, these are implemented using code like this:
8877 push throw_pos - method
8878 call <exception trampoline>
8880 The problematic part is push throw_pos - method, which cannot be represented
8881 in the LLVM IR, since it does not support label values.
8882 -> this can be implemented in AOT mode using inline asm + labels, but cannot
8883 be implemented in JIT mode ?
8884 -> a possible but slower implementation would use the normal exception
8885 throwing code but it would need to control the placement of the throw code
8886 (it needs to be exactly after the compare+branch).
8887 -> perhaps add a PC offset intrinsics ?
8889 - efficient implementation of .ovf opcodes.
8891 These are currently implemented as:
8892 <ins which sets the condition codes>
8895 Some overflow opcodes are now supported by LLVM SVN.
8897 - exception handling, unwinding.
8898 - SSA is disabled for methods with exception handlers
8899 - How to obtain unwind info for LLVM compiled methods ?
8900 -> this is now solved by converting the unwind info generated by LLVM
8902 - LLVM uses the c++ exception handling framework, while we use our home grown
8903 code, and couldn't use the c++ one:
8904 - its not supported under VC++, other exotic platforms.
8905 - it might be impossible to support filter clauses with it.
8909 The trampolines need a predictable call sequence, since they need to disasm
8910 the calling code to obtain register numbers / offsets.
8912 LLVM currently generates this code in non-JIT mode:
8913 mov -0x98(%rax),%eax
8915 Here, the vtable pointer is lost.
8916 -> solution: use one vtable trampoline per class.
8918 - passing/receiving the IMT pointer/RGCTX.
8919 -> solution: pass them as normal arguments ?
8923 LLVM does not allow the specification of argument registers etc. This means
8924 that all calls are made according to the platform ABI.
8926 - passing/receiving vtypes.
8928 Vtypes passed/received in registers are handled by the front end by using
8929 a signature with scalar arguments, and loading the parts of the vtype into those
8932 Vtypes passed on the stack are handled using the 'byval' attribute.
8936 Supported though alloca, we need to emit the load/store code.
8940 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
8941 typed registers, so we have to keep track of the precise LLVM type of each vreg.
8942 This is made easier because the IR is already in SSA form.
8943 An additional problem is that our IR is not consistent with types, i.e. i32/ia64
8944 types are frequently used incorrectly.
8949 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
8950 it with the file containing the methods emitted by the JIT and the AOT data
8954 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
8955 * - each bblock should end with a branch
8956 * - setting the return value, making cfg->ret non-volatile
8957 * - avoid some transformations in the JIT which make it harder for us to generate
8959 * - use pointer types to help optimizations.