2 * mini-llvm.c: llvm "Backend" for the mono JIT
4 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
5 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
9 #include <mono/metadata/debug-helpers.h>
10 #include <mono/metadata/debug-mono-symfile.h>
11 #include <mono/metadata/mempool-internals.h>
12 #include <mono/metadata/environment.h>
13 #include <mono/metadata/object-internals.h>
14 #include <mono/metadata/abi-details.h>
15 #include <mono/utils/mono-tls.h>
16 #include <mono/utils/mono-dl.h>
17 #include <mono/utils/mono-time.h>
18 #include <mono/utils/freebsd-dwarf.h>
20 #ifndef __STDC_LIMIT_MACROS
21 #define __STDC_LIMIT_MACROS
23 #ifndef __STDC_CONSTANT_MACROS
24 #define __STDC_CONSTANT_MACROS
27 #include "llvm-c/Core.h"
28 #include "llvm-c/ExecutionEngine.h"
29 #include "llvm-c/BitWriter.h"
30 #include "llvm-c/Analysis.h"
32 #include "mini-llvm-cpp.h"
33 #include "aot-compiler.h"
34 #include "mini-llvm.h"
39 extern void *memset(void *, int, size_t);
40 void bzero (void *to, size_t count) { memset (to, 0, count); }
44 #if LLVM_API_VERSION < 4
45 #error "The version of the mono llvm repository is too old."
49 * Information associated by mono with LLVM modules.
52 LLVMModuleRef lmodule;
53 LLVMValueRef throw_icall, rethrow, match_exc, throw_corlib_exception, resume_eh;
54 GHashTable *llvm_types;
56 const char *got_symbol;
57 const char *get_method_symbol;
58 const char *get_unbox_tramp_symbol;
59 GHashTable *plt_entries;
60 GHashTable *plt_entries_ji;
61 GHashTable *method_to_lmethod;
62 GHashTable *direct_callables;
67 GPtrArray *subprogram_mds;
69 LLVMExecutionEngineRef ee;
70 gboolean external_symbols;
75 MonoAssembly *assembly;
77 MonoAotFileInfo aot_info;
78 const char *jit_got_symbol;
79 const char *eh_frame_symbol;
80 LLVMValueRef get_method, get_unbox_tramp;
81 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
82 LLVMValueRef code_start, code_end;
83 LLVMValueRef inited_var;
84 int max_inited_idx, max_method_idx;
85 gboolean has_jitted_code;
88 GHashTable *idx_to_lmethod;
89 GHashTable *idx_to_unbox_tramp;
90 /* Maps a MonoMethod to LLVM instructions representing it */
91 GHashTable *method_to_callers;
92 LLVMContextRef context;
93 LLVMValueRef sentinel_exception;
97 * Information associated by the backend with mono basic blocks.
100 LLVMBasicBlockRef bblock, end_bblock;
101 LLVMValueRef finally_ind;
102 gboolean added, invoke_target;
104 * If this bblock is the start of a finally clause, this is a list of bblocks it
105 * needs to branch to in ENDFINALLY.
107 GSList *call_handler_return_bbs;
109 * If this bblock is the start of a finally clause, this is the bblock that
110 * CALL_HANDLER needs to branch to.
112 LLVMBasicBlockRef call_handler_target_bb;
113 /* The list of switch statements generated by ENDFINALLY instructions */
114 GSList *endfinally_switch_ins_list;
119 * Structure containing emit state
122 MonoMemPool *mempool;
124 /* Maps method names to the corresponding LLVMValueRef */
125 GHashTable *emitted_method_decls;
128 LLVMValueRef lmethod;
129 MonoLLVMModule *module;
130 LLVMModuleRef lmodule;
132 int sindex, default_index, ex_index;
133 LLVMBuilderRef builder;
134 LLVMValueRef *values, *addresses;
135 MonoType **vreg_cli_types;
137 MonoMethodSignature *sig;
139 GHashTable *region_to_handler;
140 GHashTable *clause_to_handler;
141 LLVMBuilderRef alloca_builder;
142 LLVMValueRef last_alloca;
143 LLVMValueRef rgctx_arg;
144 LLVMValueRef this_arg;
145 LLVMTypeRef *vreg_types;
146 LLVMTypeRef method_type;
147 LLVMBasicBlockRef init_bb, inited_bb;
149 gboolean *unreachable;
151 gboolean has_got_access;
152 gboolean is_linkonce;
153 int this_arg_pindex, rgctx_arg_pindex;
154 LLVMValueRef imt_rgctx_loc;
155 GHashTable *llvm_types;
157 MonoDebugMethodInfo *minfo;
159 /* For every clause, the clauses it is nested in */
162 GHashTable *exc_meta;
163 GHashTable *method_to_callers;
164 GPtrArray *phi_values;
165 GPtrArray *bblock_list;
172 MonoBasicBlock *in_bb;
177 * Instruction metadata
178 * This is the same as ins_info, but LREG != IREG.
186 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
187 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
194 /* keep in sync with the enum in mini.h */
197 #include "mini-ops.h"
202 #if SIZEOF_VOID_P == 4
203 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
205 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
208 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
211 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
213 #define TRACE_FAILURE(msg)
217 #define IS_TARGET_X86 1
219 #define IS_TARGET_X86 0
223 #define IS_TARGET_AMD64 1
225 #define IS_TARGET_AMD64 0
228 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
230 static LLVMIntPredicate cond_to_llvm_cond [] = {
243 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
256 static MonoNativeTlsKey current_cfg_tls_id;
258 static MonoLLVMModule aot_module;
259 static int memset_param_count, memcpy_param_count;
260 static const char *memset_func_name;
261 static const char *memcpy_func_name;
263 static void init_jit_module (MonoDomain *domain);
265 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
266 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
267 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
268 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
271 set_failure (EmitContext *ctx, const char *message)
273 TRACE_FAILURE (reason);
274 ctx->cfg->exception_message = g_strdup (message);
275 ctx->cfg->disable_llvm = TRUE;
281 * The LLVM type with width == sizeof (gpointer)
286 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
292 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
298 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
304 * Return the size of the LLVM representation of the vtype T.
307 get_vtype_size (MonoType *t)
311 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
313 /* LLVMArgAsIArgs depends on this since it stores whole words */
314 while (size < 2 * sizeof (gpointer) && mono_is_power_of_two (size) == -1)
321 * simd_class_to_llvm_type:
323 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
326 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
328 if (!strcmp (klass->name, "Vector2d")) {
329 return LLVMVectorType (LLVMDoubleType (), 2);
330 } else if (!strcmp (klass->name, "Vector2l")) {
331 return LLVMVectorType (LLVMInt64Type (), 2);
332 } else if (!strcmp (klass->name, "Vector2ul")) {
333 return LLVMVectorType (LLVMInt64Type (), 2);
334 } else if (!strcmp (klass->name, "Vector4i")) {
335 return LLVMVectorType (LLVMInt32Type (), 4);
336 } else if (!strcmp (klass->name, "Vector4ui")) {
337 return LLVMVectorType (LLVMInt32Type (), 4);
338 } else if (!strcmp (klass->name, "Vector4f")) {
339 return LLVMVectorType (LLVMFloatType (), 4);
340 } else if (!strcmp (klass->name, "Vector8s")) {
341 return LLVMVectorType (LLVMInt16Type (), 8);
342 } else if (!strcmp (klass->name, "Vector8us")) {
343 return LLVMVectorType (LLVMInt16Type (), 8);
344 } else if (!strcmp (klass->name, "Vector16sb")) {
345 return LLVMVectorType (LLVMInt8Type (), 16);
346 } else if (!strcmp (klass->name, "Vector16b")) {
347 return LLVMVectorType (LLVMInt8Type (), 16);
349 printf ("%s\n", klass->name);
355 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
356 static inline G_GNUC_UNUSED LLVMTypeRef
357 type_to_simd_type (int type)
361 return LLVMVectorType (LLVMInt8Type (), 16);
363 return LLVMVectorType (LLVMInt16Type (), 8);
365 return LLVMVectorType (LLVMInt32Type (), 4);
367 return LLVMVectorType (LLVMInt64Type (), 2);
369 return LLVMVectorType (LLVMDoubleType (), 2);
371 return LLVMVectorType (LLVMFloatType (), 4);
373 g_assert_not_reached ();
379 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
381 int i, size, nfields, esize;
382 LLVMTypeRef *eltypes;
387 t = &klass->byval_arg;
389 if (mini_type_is_hfa (t, &nfields, &esize)) {
391 * This is needed on arm64 where HFAs are returned in
395 eltypes = g_new (LLVMTypeRef, size);
396 for (i = 0; i < size; ++i)
397 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
399 size = get_vtype_size (t);
401 eltypes = g_new (LLVMTypeRef, size);
402 for (i = 0; i < size; ++i)
403 eltypes [i] = LLVMInt8Type ();
406 name = mono_type_full_name (&klass->byval_arg);
407 ltype = LLVMStructCreateNamed (module->context, name);
408 LLVMStructSetBody (ltype, eltypes, size, FALSE);
418 * Return the LLVM type corresponding to T.
421 type_to_llvm_type (EmitContext *ctx, MonoType *t)
423 t = mini_get_underlying_type (t);
427 return LLVMVoidType ();
429 return LLVMInt8Type ();
431 return LLVMInt16Type ();
433 return LLVMInt32Type ();
435 return LLVMInt8Type ();
437 return LLVMInt16Type ();
439 return LLVMInt32Type ();
440 case MONO_TYPE_BOOLEAN:
441 return LLVMInt8Type ();
444 return LLVMInt64Type ();
446 return LLVMInt16Type ();
448 return LLVMFloatType ();
450 return LLVMDoubleType ();
453 return IntPtrType ();
454 case MONO_TYPE_OBJECT:
455 case MONO_TYPE_CLASS:
456 case MONO_TYPE_ARRAY:
457 case MONO_TYPE_SZARRAY:
458 case MONO_TYPE_STRING:
460 return ObjRefType ();
463 /* Because of generic sharing */
464 return ObjRefType ();
465 case MONO_TYPE_GENERICINST:
466 if (!mono_type_generic_inst_is_valuetype (t))
467 return ObjRefType ();
469 case MONO_TYPE_VALUETYPE:
470 case MONO_TYPE_TYPEDBYREF: {
474 klass = mono_class_from_mono_type (t);
476 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
477 return simd_class_to_llvm_type (ctx, klass);
480 return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
482 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
484 ltype = create_llvm_type_for_type (ctx->module, klass);
485 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
491 printf ("X: %d\n", t->type);
492 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
493 ctx->cfg->disable_llvm = TRUE;
501 * Return whenever T is an unsigned int type.
504 type_is_unsigned (EmitContext *ctx, MonoType *t)
506 t = mini_get_underlying_type (t);
522 * type_to_llvm_arg_type:
524 * Same as type_to_llvm_type, but treat i8/i16 as i32.
527 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
529 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
531 if (ctx->cfg->llvm_only)
535 * This works on all abis except arm64/ios which passes multiple
536 * arguments in one stack slot.
539 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
541 * LLVM generates code which only sets the lower bits, while JITted
542 * code expects all the bits to be set.
544 ptype = LLVMInt32Type ();
552 * llvm_type_to_stack_type:
554 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
557 static G_GNUC_UNUSED LLVMTypeRef
558 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
562 if (type == LLVMInt8Type ())
563 return LLVMInt32Type ();
564 else if (type == LLVMInt16Type ())
565 return LLVMInt32Type ();
566 else if (!cfg->r4fp && type == LLVMFloatType ())
567 return LLVMDoubleType ();
573 * regtype_to_llvm_type:
575 * Return the LLVM type corresponding to the regtype C used in instruction
579 regtype_to_llvm_type (char c)
583 return LLVMInt32Type ();
585 return LLVMInt64Type ();
587 return LLVMDoubleType ();
596 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
599 op_to_llvm_type (int opcode)
604 return LLVMInt8Type ();
607 return LLVMInt8Type ();
610 return LLVMInt16Type ();
613 return LLVMInt16Type ();
616 return LLVMInt32Type ();
619 return LLVMInt32Type ();
621 return LLVMInt64Type ();
623 return LLVMFloatType ();
625 return LLVMDoubleType ();
627 return LLVMInt64Type ();
629 return LLVMInt32Type ();
631 return LLVMInt64Type ();
636 return LLVMInt8Type ();
641 return LLVMInt16Type ();
643 return LLVMInt32Type ();
646 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
653 return LLVMInt32Type ();
660 return LLVMInt64Type ();
662 printf ("%s\n", mono_inst_name (opcode));
663 g_assert_not_reached ();
668 #define CLAUSE_START(clause) ((clause)->try_offset)
669 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
672 * load_store_to_llvm_type:
674 * Return the size/sign/zero extension corresponding to the load/store opcode
678 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
684 case OP_LOADI1_MEMBASE:
685 case OP_STOREI1_MEMBASE_REG:
686 case OP_STOREI1_MEMBASE_IMM:
687 case OP_ATOMIC_LOAD_I1:
688 case OP_ATOMIC_STORE_I1:
691 return LLVMInt8Type ();
692 case OP_LOADU1_MEMBASE:
694 case OP_ATOMIC_LOAD_U1:
695 case OP_ATOMIC_STORE_U1:
698 return LLVMInt8Type ();
699 case OP_LOADI2_MEMBASE:
700 case OP_STOREI2_MEMBASE_REG:
701 case OP_STOREI2_MEMBASE_IMM:
702 case OP_ATOMIC_LOAD_I2:
703 case OP_ATOMIC_STORE_I2:
706 return LLVMInt16Type ();
707 case OP_LOADU2_MEMBASE:
709 case OP_ATOMIC_LOAD_U2:
710 case OP_ATOMIC_STORE_U2:
713 return LLVMInt16Type ();
714 case OP_LOADI4_MEMBASE:
715 case OP_LOADU4_MEMBASE:
718 case OP_STOREI4_MEMBASE_REG:
719 case OP_STOREI4_MEMBASE_IMM:
720 case OP_ATOMIC_LOAD_I4:
721 case OP_ATOMIC_STORE_I4:
722 case OP_ATOMIC_LOAD_U4:
723 case OP_ATOMIC_STORE_U4:
725 return LLVMInt32Type ();
726 case OP_LOADI8_MEMBASE:
728 case OP_STOREI8_MEMBASE_REG:
729 case OP_STOREI8_MEMBASE_IMM:
730 case OP_ATOMIC_LOAD_I8:
731 case OP_ATOMIC_STORE_I8:
732 case OP_ATOMIC_LOAD_U8:
733 case OP_ATOMIC_STORE_U8:
735 return LLVMInt64Type ();
736 case OP_LOADR4_MEMBASE:
737 case OP_STORER4_MEMBASE_REG:
738 case OP_ATOMIC_LOAD_R4:
739 case OP_ATOMIC_STORE_R4:
741 return LLVMFloatType ();
742 case OP_LOADR8_MEMBASE:
743 case OP_STORER8_MEMBASE_REG:
744 case OP_ATOMIC_LOAD_R8:
745 case OP_ATOMIC_STORE_R8:
747 return LLVMDoubleType ();
748 case OP_LOAD_MEMBASE:
750 case OP_STORE_MEMBASE_REG:
751 case OP_STORE_MEMBASE_IMM:
752 *size = sizeof (gpointer);
753 return IntPtrType ();
755 g_assert_not_reached ();
763 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
766 ovf_op_to_intrins (int opcode)
770 return "llvm.sadd.with.overflow.i32";
772 return "llvm.uadd.with.overflow.i32";
774 return "llvm.ssub.with.overflow.i32";
776 return "llvm.usub.with.overflow.i32";
778 return "llvm.smul.with.overflow.i32";
780 return "llvm.umul.with.overflow.i32";
782 return "llvm.sadd.with.overflow.i64";
784 return "llvm.uadd.with.overflow.i64";
786 return "llvm.ssub.with.overflow.i64";
788 return "llvm.usub.with.overflow.i64";
790 return "llvm.smul.with.overflow.i64";
792 return "llvm.umul.with.overflow.i64";
794 g_assert_not_reached ();
800 simd_op_to_intrins (int opcode)
803 #if defined(TARGET_X86) || defined(TARGET_AMD64)
805 return "llvm.x86.sse2.min.pd";
807 return "llvm.x86.sse.min.ps";
809 return "llvm.x86.sse41.pminud";
811 return "llvm.x86.sse41.pminuw";
813 return "llvm.x86.sse2.pminu.b";
815 return "llvm.x86.sse2.pmins.w";
817 return "llvm.x86.sse2.max.pd";
819 return "llvm.x86.sse.max.ps";
821 return "llvm.x86.sse3.hadd.pd";
823 return "llvm.x86.sse3.hadd.ps";
825 return "llvm.x86.sse3.hsub.pd";
827 return "llvm.x86.sse3.hsub.ps";
829 return "llvm.x86.sse41.pmaxud";
831 return "llvm.x86.sse41.pmaxuw";
833 return "llvm.x86.sse2.pmaxu.b";
835 return "llvm.x86.sse3.addsub.ps";
837 return "llvm.x86.sse3.addsub.pd";
838 case OP_EXTRACT_MASK:
839 return "llvm.x86.sse2.pmovmskb.128";
842 return "llvm.x86.sse2.psrli.w";
845 return "llvm.x86.sse2.psrli.d";
848 return "llvm.x86.sse2.psrli.q";
851 return "llvm.x86.sse2.pslli.w";
854 return "llvm.x86.sse2.pslli.d";
857 return "llvm.x86.sse2.pslli.q";
860 return "llvm.x86.sse2.psrai.w";
863 return "llvm.x86.sse2.psrai.d";
865 return "llvm.x86.sse2.padds.b";
867 return "llvm.x86.sse2.padds.w";
869 return "llvm.x86.sse2.psubs.b";
871 return "llvm.x86.sse2.psubs.w";
872 case OP_PADDB_SAT_UN:
873 return "llvm.x86.sse2.paddus.b";
874 case OP_PADDW_SAT_UN:
875 return "llvm.x86.sse2.paddus.w";
876 case OP_PSUBB_SAT_UN:
877 return "llvm.x86.sse2.psubus.b";
878 case OP_PSUBW_SAT_UN:
879 return "llvm.x86.sse2.psubus.w";
881 return "llvm.x86.sse2.pavg.b";
883 return "llvm.x86.sse2.pavg.w";
885 return "llvm.x86.sse.sqrt.ps";
887 return "llvm.x86.sse2.sqrt.pd";
889 return "llvm.x86.sse.rsqrt.ps";
891 return "llvm.x86.sse.rcp.ps";
893 return "llvm.x86.sse2.cvtdq2pd";
895 return "llvm.x86.sse2.cvtdq2ps";
897 return "llvm.x86.sse2.cvtpd2dq";
899 return "llvm.x86.sse2.cvtps2dq";
901 return "llvm.x86.sse2.cvtpd2ps";
903 return "llvm.x86.sse2.cvtps2pd";
905 return "llvm.x86.sse2.cvttpd2dq";
907 return "llvm.x86.sse2.cvttps2dq";
909 return "llvm.x86.sse.cmp.ps";
911 return "llvm.x86.sse2.cmp.pd";
913 return "llvm.x86.sse2.packsswb.128";
915 return "llvm.x86.sse2.packssdw.128";
917 return "llvm.x86.sse2.packuswb.128";
919 return "llvm.x86.sse41.packusdw";
921 return "llvm.x86.sse2.pmulh.w";
922 case OP_PMULW_HIGH_UN:
923 return "llvm.x86.sse2.pmulhu.w";
926 g_assert_not_reached ();
932 simd_op_to_llvm_type (int opcode)
934 #if defined(TARGET_X86) || defined(TARGET_AMD64)
938 return type_to_simd_type (MONO_TYPE_R8);
941 return type_to_simd_type (MONO_TYPE_I8);
944 return type_to_simd_type (MONO_TYPE_I4);
949 return type_to_simd_type (MONO_TYPE_I2);
953 return type_to_simd_type (MONO_TYPE_I1);
955 return type_to_simd_type (MONO_TYPE_R4);
958 return type_to_simd_type (MONO_TYPE_I4);
962 return type_to_simd_type (MONO_TYPE_R8);
966 return type_to_simd_type (MONO_TYPE_R4);
967 case OP_EXTRACT_MASK:
968 return type_to_simd_type (MONO_TYPE_I1);
974 return type_to_simd_type (MONO_TYPE_R4);
977 return type_to_simd_type (MONO_TYPE_R8);
979 g_assert_not_reached ();
990 * Return the LLVM basic block corresponding to BB.
992 static LLVMBasicBlockRef
993 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
995 char bb_name_buf [128];
998 if (ctx->bblocks [bb->block_num].bblock == NULL) {
999 if (bb->flags & BB_EXCEPTION_HANDLER) {
1000 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
1001 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
1002 bb_name = bb_name_buf;
1003 } else if (bb->block_num < 256) {
1004 if (!ctx->module->bb_names) {
1005 ctx->module->bb_names_len = 256;
1006 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
1008 if (!ctx->module->bb_names [bb->block_num]) {
1011 n = g_strdup_printf ("BB%d", bb->block_num);
1012 mono_memory_barrier ();
1013 ctx->module->bb_names [bb->block_num] = n;
1015 bb_name = ctx->module->bb_names [bb->block_num];
1017 sprintf (bb_name_buf, "BB%d", bb->block_num);
1018 bb_name = bb_name_buf;
1021 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1022 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1025 return ctx->bblocks [bb->block_num].bblock;
1031 * Return the last LLVM bblock corresponding to BB.
1032 * This might not be equal to the bb returned by get_bb () since we need to generate
1033 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1035 static LLVMBasicBlockRef
1036 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1039 return ctx->bblocks [bb->block_num].end_bblock;
1042 static LLVMBasicBlockRef
1043 gen_bb (EmitContext *ctx, const char *prefix)
1047 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1048 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1054 * Return the target of the patch identified by TYPE and TARGET.
1057 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1063 memset (&ji, 0, sizeof (ji));
1065 ji.data.target = target;
1067 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, &error);
1068 mono_error_assert_ok (&error);
1076 * Emit code to convert the LLVM value V to DTYPE.
1079 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1081 LLVMTypeRef stype = LLVMTypeOf (v);
1083 if (stype != dtype) {
1084 gboolean ext = FALSE;
1087 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1089 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1091 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1095 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1097 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1098 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1101 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1102 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1103 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1104 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1105 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1106 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1107 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1108 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1110 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1111 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1112 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1113 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1114 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1115 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1117 if (mono_arch_is_soft_float ()) {
1118 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1119 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1120 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1121 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1124 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1125 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1128 LLVMDumpValue (LLVMConstNull (dtype));
1129 g_assert_not_reached ();
1137 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1139 return convert_full (ctx, v, dtype, FALSE);
1143 * emit_volatile_load:
1145 * If vreg is volatile, emit a load from its address.
1148 emit_volatile_load (EmitContext *ctx, int vreg)
1152 LLVMValueRef v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1153 t = ctx->vreg_cli_types [vreg];
1154 if (t && !t->byref) {
1156 * Might have to zero extend since llvm doesn't have
1159 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1160 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1161 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1162 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1163 else if (t->type == MONO_TYPE_U8)
1164 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1171 * emit_volatile_store:
1173 * If VREG is volatile, emit a store from its value to its address.
1176 emit_volatile_store (EmitContext *ctx, int vreg)
1178 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1180 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1181 g_assert (ctx->addresses [vreg]);
1182 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1187 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1189 LLVMTypeRef ret_type;
1190 LLVMTypeRef *param_types = NULL;
1195 rtype = mini_get_underlying_type (sig->ret);
1196 ret_type = type_to_llvm_type (ctx, rtype);
1200 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1204 param_types [pindex ++] = ThisType ();
1205 for (i = 0; i < sig->param_count; ++i)
1206 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1208 if (!ctx_ok (ctx)) {
1209 g_free (param_types);
1213 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1214 g_free (param_types);
1220 * sig_to_llvm_sig_full:
1222 * Return the LLVM signature corresponding to the mono signature SIG using the
1223 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1226 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1228 LLVMTypeRef ret_type;
1229 LLVMTypeRef *param_types = NULL;
1231 int i, j, pindex, vret_arg_pindex = 0;
1232 gboolean vretaddr = FALSE;
1236 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1238 rtype = mini_get_underlying_type (sig->ret);
1239 ret_type = type_to_llvm_type (ctx, rtype);
1243 switch (cinfo->ret.storage) {
1244 case LLVMArgVtypeInReg:
1245 /* LLVM models this by returning an aggregate value */
1246 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1247 LLVMTypeRef members [2];
1249 members [0] = IntPtrType ();
1250 ret_type = LLVMStructType (members, 1, FALSE);
1251 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1253 ret_type = LLVMVoidType ();
1254 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1255 LLVMTypeRef members [2];
1257 members [0] = IntPtrType ();
1258 members [1] = IntPtrType ();
1259 ret_type = LLVMStructType (members, 2, FALSE);
1261 g_assert_not_reached ();
1264 case LLVMArgVtypeByVal:
1265 /* Vtype returned normally by val */
1267 case LLVMArgVtypeAsScalar: {
1268 int size = mono_class_value_size (mono_class_from_mono_type (rtype), NULL);
1269 /* LLVM models this by returning an int */
1270 if (size < SIZEOF_VOID_P) {
1271 g_assert (cinfo->ret.nslots == 1);
1272 ret_type = LLVMIntType (size * 8);
1274 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1275 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (mgreg_t) * 8);
1279 case LLVMArgFpStruct: {
1280 /* Vtype returned as a fp struct */
1281 LLVMTypeRef members [16];
1283 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1284 for (i = 0; i < cinfo->ret.nslots; ++i)
1285 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1286 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1289 case LLVMArgVtypeByRef:
1290 /* Vtype returned using a hidden argument */
1291 ret_type = LLVMVoidType ();
1293 case LLVMArgVtypeRetAddr:
1294 case LLVMArgScalarRetAddr:
1295 case LLVMArgGsharedvtFixed:
1296 case LLVMArgGsharedvtFixedVtype:
1297 case LLVMArgGsharedvtVariable:
1299 ret_type = LLVMVoidType ();
1305 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1307 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1309 * Has to be the first argument because of the sret argument attribute
1310 * FIXME: This might conflict with passing 'this' as the first argument, but
1311 * this is only used on arm64 which has a dedicated struct return register.
1313 cinfo->vret_arg_pindex = pindex;
1314 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1315 if (!ctx_ok (ctx)) {
1316 g_free (param_types);
1319 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1322 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1323 cinfo->rgctx_arg_pindex = pindex;
1324 param_types [pindex] = ctx->module->ptr_type;
1327 if (cinfo->imt_arg) {
1328 cinfo->imt_arg_pindex = pindex;
1329 param_types [pindex] = ctx->module->ptr_type;
1333 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1334 vret_arg_pindex = pindex;
1335 if (cinfo->vret_arg_index == 1) {
1336 /* Add the slots consumed by the first argument */
1337 LLVMArgInfo *ainfo = &cinfo->args [0];
1338 switch (ainfo->storage) {
1339 case LLVMArgVtypeInReg:
1340 for (j = 0; j < 2; ++j) {
1341 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1350 cinfo->vret_arg_pindex = vret_arg_pindex;
1353 if (vretaddr && vret_arg_pindex == pindex)
1354 param_types [pindex ++] = IntPtrType ();
1356 cinfo->this_arg_pindex = pindex;
1357 param_types [pindex ++] = ThisType ();
1358 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1360 if (vretaddr && vret_arg_pindex == pindex)
1361 param_types [pindex ++] = IntPtrType ();
1362 for (i = 0; i < sig->param_count; ++i) {
1363 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1365 if (vretaddr && vret_arg_pindex == pindex)
1366 param_types [pindex ++] = IntPtrType ();
1367 ainfo->pindex = pindex;
1369 switch (ainfo->storage) {
1370 case LLVMArgVtypeInReg:
1371 for (j = 0; j < 2; ++j) {
1372 switch (ainfo->pair_storage [j]) {
1374 param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
1379 g_assert_not_reached ();
1383 case LLVMArgVtypeByVal:
1384 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1387 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1390 case LLVMArgAsIArgs:
1391 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1394 case LLVMArgVtypeByRef:
1395 case LLVMArgScalarByRef:
1396 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1399 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1402 case LLVMArgAsFpArgs: {
1405 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1406 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1407 param_types [pindex ++] = LLVMDoubleType ();
1408 for (j = 0; j < ainfo->nslots; ++j)
1409 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1412 case LLVMArgVtypeAsScalar:
1413 g_assert_not_reached ();
1415 case LLVMArgGsharedvtFixed:
1416 case LLVMArgGsharedvtFixedVtype:
1417 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1419 case LLVMArgGsharedvtVariable:
1420 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1423 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1427 if (!ctx_ok (ctx)) {
1428 g_free (param_types);
1431 if (vretaddr && vret_arg_pindex == pindex)
1432 param_types [pindex ++] = IntPtrType ();
1433 if (ctx->llvm_only && cinfo->rgctx_arg) {
1434 /* Pass the rgctx as the last argument */
1435 cinfo->rgctx_arg_pindex = pindex;
1436 param_types [pindex] = ctx->module->ptr_type;
1440 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1441 g_free (param_types);
1447 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1449 return sig_to_llvm_sig_full (ctx, sig, NULL);
1453 * LLVMFunctionType1:
1455 * Create an LLVM function type from the arguments.
1457 static G_GNUC_UNUSED LLVMTypeRef
1458 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1461 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1465 * LLVMFunctionType1:
1467 * Create an LLVM function type from the arguments.
1469 static G_GNUC_UNUSED LLVMTypeRef
1470 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1471 LLVMTypeRef ParamType1,
1474 LLVMTypeRef param_types [1];
1476 param_types [0] = ParamType1;
1478 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1482 * LLVMFunctionType2:
1484 * Create an LLVM function type from the arguments.
1486 static G_GNUC_UNUSED LLVMTypeRef
1487 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1488 LLVMTypeRef ParamType1,
1489 LLVMTypeRef ParamType2,
1492 LLVMTypeRef param_types [2];
1494 param_types [0] = ParamType1;
1495 param_types [1] = ParamType2;
1497 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1501 * LLVMFunctionType3:
1503 * Create an LLVM function type from the arguments.
1505 static G_GNUC_UNUSED LLVMTypeRef
1506 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1507 LLVMTypeRef ParamType1,
1508 LLVMTypeRef ParamType2,
1509 LLVMTypeRef ParamType3,
1512 LLVMTypeRef param_types [3];
1514 param_types [0] = ParamType1;
1515 param_types [1] = ParamType2;
1516 param_types [2] = ParamType3;
1518 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1521 static G_GNUC_UNUSED LLVMTypeRef
1522 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1523 LLVMTypeRef ParamType1,
1524 LLVMTypeRef ParamType2,
1525 LLVMTypeRef ParamType3,
1526 LLVMTypeRef ParamType4,
1527 LLVMTypeRef ParamType5,
1530 LLVMTypeRef param_types [5];
1532 param_types [0] = ParamType1;
1533 param_types [1] = ParamType2;
1534 param_types [2] = ParamType3;
1535 param_types [3] = ParamType4;
1536 param_types [4] = ParamType5;
1538 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1544 * Create an LLVM builder and remember it so it can be freed later.
1546 static LLVMBuilderRef
1547 create_builder (EmitContext *ctx)
1549 LLVMBuilderRef builder = LLVMCreateBuilder ();
1551 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1557 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1562 case MONO_PATCH_INFO_INTERNAL_METHOD:
1563 name = g_strdup_printf ("jit_icall_%s", data);
1565 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1566 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1567 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1571 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1579 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1583 LLVMValueRef indexes [2];
1585 LLVMValueRef got_entry_addr, load;
1586 LLVMBuilderRef builder = ctx->builder;
1591 ji = g_new0 (MonoJumpInfo, 1);
1593 ji->data.target = data;
1595 ji = mono_aot_patch_info_dup (ji);
1597 ji->next = cfg->patch_info;
1598 cfg->patch_info = ji;
1600 got_offset = mono_aot_get_got_offset (cfg->patch_info);
1601 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1603 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1604 * explicitly initialize it.
1606 if (!mono_aot_is_shared_got_offset (got_offset)) {
1607 //mono_print_ji (ji);
1609 ctx->has_got_access = TRUE;
1612 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1613 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1614 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1616 name = get_aotconst_name (type, data, got_offset);
1618 load = LLVMBuildLoad (builder, got_entry_addr, "");
1619 load = convert (ctx, load, llvm_type);
1620 LLVMSetValueName (load, name ? name : "");
1622 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1625 //set_invariant_load_flag (load);
1631 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1633 return get_aotconst_typed (ctx, type, data, NULL);
1637 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1639 LLVMValueRef callee;
1641 if (ctx->llvm_only) {
1642 callee_name = mono_aot_get_direct_call_symbol (type, data);
1644 /* Directly callable */
1646 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1648 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1650 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1652 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1654 /* LLVMTypeRef's are uniqued */
1655 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1656 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1658 g_free (callee_name);
1664 * Calls are made through the GOT.
1666 return get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
1668 MonoJumpInfo *ji = NULL;
1670 callee_name = mono_aot_get_plt_symbol (type, data);
1674 if (ctx->cfg->compile_aot)
1675 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1676 mono_add_patch_info (ctx->cfg, 0, type, data);
1679 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
1681 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1683 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1685 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
1688 if (ctx->cfg->compile_aot) {
1689 ji = g_new0 (MonoJumpInfo, 1);
1691 ji->data.target = data;
1693 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
1701 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
1703 MonoMethodHeader *header = cfg->header;
1704 MonoExceptionClause *clause;
1708 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
1709 return (bb->region >> 8) - 1;
1712 for (i = 0; i < header->num_clauses; ++i) {
1713 clause = &header->clauses [i];
1715 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
1722 static MonoExceptionClause *
1723 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
1725 // Since they're sorted by nesting we just need
1726 // the first one that the bb is a member of
1727 MonoExceptionClause *last = NULL;
1729 for (int i = 0; i < cfg->header->num_clauses; i++) {
1730 MonoExceptionClause *curr = &cfg->header->clauses [i];
1732 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
1735 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset)) {
1736 if (last && CLAUSE_END(last) > CLAUSE_END(curr))
1750 set_metadata_flag (LLVMValueRef v, const char *flag_name)
1752 LLVMValueRef md_arg;
1755 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1756 md_arg = LLVMMDString ("mono", 4);
1757 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1761 set_invariant_load_flag (LLVMValueRef v)
1763 LLVMValueRef md_arg;
1765 const char *flag_name;
1767 // FIXME: Cache this
1768 flag_name = "invariant.load";
1769 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1770 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
1771 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1777 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
1781 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
1783 MonoCompile *cfg = ctx->cfg;
1784 LLVMValueRef lcall = NULL;
1785 LLVMBuilderRef builder = *builder_ref;
1786 MonoExceptionClause *clause;
1788 if (ctx->llvm_only) {
1789 clause = get_most_deep_clause (cfg, ctx, bb);
1792 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1795 * Have to use an invoke instead of a call, branching to the
1796 * handler bblock of the clause containing this bblock.
1798 intptr_t key = CLAUSE_END(clause);
1800 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
1802 // FIXME: Find the one that has the lowest end bound for the right start address
1803 // FIXME: Finally + nesting
1806 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
1809 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
1811 builder = ctx->builder = create_builder (ctx);
1812 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1814 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1818 int clause_index = get_handler_clause (cfg, bb);
1820 if (clause_index != -1) {
1821 MonoMethodHeader *header = cfg->header;
1822 MonoExceptionClause *ec = &header->clauses [clause_index];
1823 MonoBasicBlock *tblock;
1824 LLVMBasicBlockRef ex_bb, noex_bb;
1827 * Have to use an invoke instead of a call, branching to the
1828 * handler bblock of the clause containing this bblock.
1831 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1833 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
1836 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
1838 ex_bb = get_bb (ctx, tblock);
1840 noex_bb = gen_bb (ctx, "NOEX_BB");
1843 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1845 builder = ctx->builder = create_builder (ctx);
1846 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1848 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1853 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1854 ctx->builder = builder;
1858 *builder_ref = ctx->builder;
1864 emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting, BarrierKind barrier)
1866 const char *intrins_name;
1867 LLVMValueRef args [16], res;
1868 LLVMTypeRef addr_type;
1869 gboolean use_intrinsics = TRUE;
1871 #if LLVM_API_VERSION > 100
1872 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1873 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
1874 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, addr, LLVMConstNull (LLVMTypeOf (addr)), "");
1875 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
1876 *builder_ref = ctx->builder;
1877 use_intrinsics = FALSE;
1881 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
1882 LLVMAtomicOrdering ordering;
1885 case LLVM_BARRIER_NONE:
1886 ordering = LLVMAtomicOrderingNotAtomic;
1888 case LLVM_BARRIER_ACQ:
1889 ordering = LLVMAtomicOrderingAcquire;
1891 case LLVM_BARRIER_SEQ:
1892 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1895 g_assert_not_reached ();
1900 * We handle loads which can fault by calling a mono specific intrinsic
1901 * using an invoke, so they are handled properly inside try blocks.
1902 * We can't use this outside clauses, since LLVM optimizes intrinsics which
1903 * are marked with IntrReadArgMem.
1907 intrins_name = "llvm.mono.load.i8.p0i8";
1910 intrins_name = "llvm.mono.load.i16.p0i16";
1913 intrins_name = "llvm.mono.load.i32.p0i32";
1916 intrins_name = "llvm.mono.load.i64.p0i64";
1919 g_assert_not_reached ();
1922 addr_type = LLVMTypeOf (addr);
1923 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
1924 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1927 args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1928 args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1929 args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1930 res = emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->lmodule, intrins_name), args, 4);
1932 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
1933 res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
1934 else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
1935 res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
1942 * We emit volatile loads for loads which can fault, because otherwise
1943 * LLVM will generate invalid code when encountering a load from a
1946 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting, barrier);
1948 /* Mark it with a custom metadata */
1951 set_metadata_flag (res, "mono.faulting.load");
1959 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
1961 return emit_load_general (ctx, bb, builder_ref, size, addr, name, is_faulting, LLVM_BARRIER_NONE);
1965 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting, BarrierKind barrier)
1967 const char *intrins_name;
1968 LLVMValueRef args [16];
1969 gboolean use_intrinsics = TRUE;
1971 #if LLVM_API_VERSION > 100
1972 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1973 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
1974 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, addr, LLVMConstNull (LLVMTypeOf (addr)), "");
1975 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
1976 *builder_ref = ctx->builder;
1977 use_intrinsics = FALSE;
1981 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
1982 LLVMAtomicOrdering ordering;
1985 case LLVM_BARRIER_NONE:
1986 ordering = LLVMAtomicOrderingNotAtomic;
1988 case LLVM_BARRIER_REL:
1989 ordering = LLVMAtomicOrderingRelease;
1991 case LLVM_BARRIER_SEQ:
1992 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1995 g_assert_not_reached ();
2001 intrins_name = "llvm.mono.store.i8.p0i8";
2004 intrins_name = "llvm.mono.store.i16.p0i16";
2007 intrins_name = "llvm.mono.store.i32.p0i32";
2010 intrins_name = "llvm.mono.store.i64.p0i64";
2013 g_assert_not_reached ();
2016 if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
2017 value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
2018 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
2023 args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2024 args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
2025 args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
2026 emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->lmodule, intrins_name), args, 5);
2028 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
2033 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting)
2035 emit_store_general (ctx, bb, builder_ref, size, value, addr, is_faulting, LLVM_BARRIER_NONE);
2039 * emit_cond_system_exception:
2041 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2042 * Might set the ctx exception.
2045 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
2047 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2048 LLVMBuilderRef builder;
2049 MonoClass *exc_class;
2050 LLVMValueRef args [2];
2051 LLVMValueRef callee;
2053 ex_bb = gen_bb (ctx, "EX_BB");
2055 ex2_bb = gen_bb (ctx, "EX2_BB");
2056 noex_bb = gen_bb (ctx, "NOEX_BB");
2058 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2060 exc_class = mono_class_from_name (mono_get_corlib (), "System", exc_type);
2061 g_assert (exc_class);
2063 /* Emit exception throwing code */
2064 ctx->builder = builder = create_builder (ctx);
2065 LLVMPositionBuilderAtEnd (builder, ex_bb);
2067 if (ctx->cfg->llvm_only) {
2068 static LLVMTypeRef sig;
2071 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2072 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, "mono_llvm_throw_corlib_exception");
2074 LLVMBuildBr (builder, ex2_bb);
2076 ctx->builder = builder = create_builder (ctx);
2077 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2079 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2080 emit_call (ctx, bb, &builder, callee, args, 1);
2081 LLVMBuildUnreachable (builder);
2083 ctx->builder = builder = create_builder (ctx);
2084 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2086 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2092 callee = ctx->module->throw_corlib_exception;
2095 const char *icall_name;
2097 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2098 icall_name = "llvm_throw_corlib_exception_abs_trampoline";
2100 if (ctx->cfg->compile_aot) {
2101 callee = get_callee (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
2103 callee = LLVMAddFunction (ctx->lmodule, "llvm_throw_corlib_exception_trampoline", sig);
2106 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2107 * - On x86, LLVM generated code doesn't push the arguments
2108 * - The trampoline takes the throw address as an arguments, not a pc offset.
2110 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
2112 mono_memory_barrier ();
2113 ctx->module->throw_corlib_exception = callee;
2117 if (IS_TARGET_X86 || IS_TARGET_AMD64)
2118 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2120 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE);
2123 * The LLVM mono branch contains changes so a block address can be passed as an
2124 * argument to a call.
2126 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2127 emit_call (ctx, bb, &builder, callee, args, 2);
2129 LLVMBuildUnreachable (builder);
2131 ctx->builder = builder = create_builder (ctx);
2132 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2134 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2141 * emit_args_to_vtype:
2143 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2146 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2148 int j, size, nslots;
2150 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
2152 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2153 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2156 if (ainfo->storage == LLVMArgAsFpArgs)
2157 nslots = ainfo->nslots;
2161 for (j = 0; j < nslots; ++j) {
2162 LLVMValueRef index [2], addr, daddr;
2163 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2164 LLVMTypeRef part_type;
2166 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2169 if (ainfo->pair_storage [j] == LLVMArgNone)
2172 switch (ainfo->pair_storage [j]) {
2173 case LLVMArgInIReg: {
2174 part_type = LLVMIntType (part_size * 8);
2175 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2176 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2177 addr = LLVMBuildGEP (builder, address, index, 1, "");
2179 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2180 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2181 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2183 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2186 case LLVMArgInFPReg: {
2187 LLVMTypeRef arg_type;
2189 if (ainfo->esize == 8)
2190 arg_type = LLVMDoubleType ();
2192 arg_type = LLVMFloatType ();
2194 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2195 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2196 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2197 LLVMBuildStore (builder, args [j], addr);
2203 g_assert_not_reached ();
2206 size -= sizeof (gpointer);
2211 * emit_vtype_to_args:
2213 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2214 * into ARGS, and the number of arguments into NARGS.
2217 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2220 int j, size, nslots;
2221 LLVMTypeRef arg_type;
2223 size = get_vtype_size (t);
2225 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t)))
2226 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2228 if (ainfo->storage == LLVMArgAsFpArgs)
2229 nslots = ainfo->nslots;
2232 for (j = 0; j < nslots; ++j) {
2233 LLVMValueRef index [2], addr, daddr;
2234 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2236 if (ainfo->pair_storage [j] == LLVMArgNone)
2239 switch (ainfo->pair_storage [j]) {
2241 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2242 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2243 addr = LLVMBuildGEP (builder, address, index, 1, "");
2245 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2246 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2247 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2249 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2251 case LLVMArgInFPReg:
2252 if (ainfo->esize == 8)
2253 arg_type = LLVMDoubleType ();
2255 arg_type = LLVMFloatType ();
2256 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2257 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2258 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2259 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2264 g_assert_not_reached ();
2266 size -= sizeof (gpointer);
2273 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2276 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2277 * get executed every time control reaches them.
2279 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2281 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2282 return ctx->last_alloca;
2286 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2288 return build_alloca_llvm_type_name (ctx, t, align, "");
2292 build_alloca (EmitContext *ctx, MonoType *t)
2294 MonoClass *k = mono_class_from_mono_type (t);
2297 g_assert (!mini_is_gsharedvt_variable_type (t));
2299 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2302 align = mono_class_min_align (k);
2304 /* Sometimes align is not a power of 2 */
2305 while (mono_is_power_of_two (align) == -1)
2308 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2312 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2316 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2318 MonoCompile *cfg = ctx->cfg;
2319 LLVMBuilderRef builder = ctx->builder;
2320 LLVMValueRef offset, offset_var;
2321 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2322 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2326 g_assert (info_var);
2327 g_assert (locals_var);
2329 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2331 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * sizeof (gpointer)), FALSE);
2332 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2334 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2335 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2337 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2341 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2344 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2347 module->used = g_ptr_array_sized_new (16);
2348 g_ptr_array_add (module->used, global);
2352 emit_llvm_used (MonoLLVMModule *module)
2354 LLVMModuleRef lmodule = module->lmodule;
2355 LLVMTypeRef used_type;
2356 LLVMValueRef used, *used_elem;
2362 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2363 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2364 used_elem = g_new0 (LLVMValueRef, module->used->len);
2365 for (i = 0; i < module->used->len; ++i)
2366 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2367 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2368 LLVMSetLinkage (used, LLVMAppendingLinkage);
2369 LLVMSetSection (used, "llvm.metadata");
2375 * Emit a function mapping method indexes to their code
2378 emit_get_method (MonoLLVMModule *module)
2380 LLVMModuleRef lmodule = module->lmodule;
2381 LLVMValueRef func, switch_ins, m;
2382 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2383 LLVMBasicBlockRef *bbs;
2385 LLVMBuilderRef builder;
2390 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2391 * but generating code seems safer.
2393 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2394 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2395 LLVMSetLinkage (func, LLVMExternalLinkage);
2396 LLVMSetVisibility (func, LLVMHiddenVisibility);
2397 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2398 module->get_method = func;
2400 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2403 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2404 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2405 * then we will have to find another solution.
2408 name = g_strdup_printf ("BB_CODE_START");
2409 code_start_bb = LLVMAppendBasicBlock (func, name);
2411 builder = LLVMCreateBuilder ();
2412 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2413 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2415 name = g_strdup_printf ("BB_CODE_END");
2416 code_end_bb = LLVMAppendBasicBlock (func, name);
2418 builder = LLVMCreateBuilder ();
2419 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2420 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2422 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2423 for (i = 0; i < module->max_method_idx + 1; ++i) {
2424 name = g_strdup_printf ("BB_%d", i);
2425 bb = LLVMAppendBasicBlock (func, name);
2429 builder = LLVMCreateBuilder ();
2430 LLVMPositionBuilderAtEnd (builder, bb);
2432 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2434 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2436 LLVMBuildRet (builder, LLVMConstNull (rtype));
2439 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2440 builder = LLVMCreateBuilder ();
2441 LLVMPositionBuilderAtEnd (builder, fail_bb);
2442 LLVMBuildRet (builder, LLVMConstNull (rtype));
2444 builder = LLVMCreateBuilder ();
2445 LLVMPositionBuilderAtEnd (builder, entry_bb);
2447 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2448 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2449 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2450 for (i = 0; i < module->max_method_idx + 1; ++i) {
2451 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2454 mark_as_used (module, func);
2458 * emit_get_unbox_tramp:
2460 * Emit a function mapping method indexes to their unbox trampoline
2463 emit_get_unbox_tramp (MonoLLVMModule *module)
2465 LLVMModuleRef lmodule = module->lmodule;
2466 LLVMValueRef func, switch_ins, m;
2467 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2468 LLVMBasicBlockRef *bbs;
2470 LLVMBuilderRef builder;
2474 /* Similar to emit_get_method () */
2476 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2477 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2478 LLVMSetLinkage (func, LLVMExternalLinkage);
2479 LLVMSetVisibility (func, LLVMHiddenVisibility);
2480 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2481 module->get_unbox_tramp = func;
2483 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2485 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2486 for (i = 0; i < module->max_method_idx + 1; ++i) {
2487 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2491 name = g_strdup_printf ("BB_%d", i);
2492 bb = LLVMAppendBasicBlock (func, name);
2496 builder = LLVMCreateBuilder ();
2497 LLVMPositionBuilderAtEnd (builder, bb);
2499 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2502 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2503 builder = LLVMCreateBuilder ();
2504 LLVMPositionBuilderAtEnd (builder, fail_bb);
2505 LLVMBuildRet (builder, LLVMConstNull (rtype));
2507 builder = LLVMCreateBuilder ();
2508 LLVMPositionBuilderAtEnd (builder, entry_bb);
2510 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2511 for (i = 0; i < module->max_method_idx + 1; ++i) {
2512 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2516 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2519 mark_as_used (module, func);
2522 /* Add a function to mark the beginning of LLVM code */
2524 emit_llvm_code_start (MonoLLVMModule *module)
2526 LLVMModuleRef lmodule = module->lmodule;
2528 LLVMBasicBlockRef entry_bb;
2529 LLVMBuilderRef builder;
2531 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2532 LLVMSetLinkage (func, LLVMInternalLinkage);
2533 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2534 module->code_start = func;
2535 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2536 builder = LLVMCreateBuilder ();
2537 LLVMPositionBuilderAtEnd (builder, entry_bb);
2538 LLVMBuildRetVoid (builder);
2542 emit_init_icall_wrapper (MonoLLVMModule *module, const char *name, const char *icall_name, int subtype)
2544 LLVMModuleRef lmodule = module->lmodule;
2545 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2546 LLVMBasicBlockRef entry_bb;
2547 LLVMBuilderRef builder;
2554 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
2555 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
2560 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
2561 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
2564 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
2565 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
2568 g_assert_not_reached ();
2570 LLVMSetLinkage (func, LLVMInternalLinkage);
2571 LLVMAddFunctionAttr (func, LLVMNoInlineAttribute);
2572 mono_llvm_set_preserveall_cc (func);
2573 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2574 builder = LLVMCreateBuilder ();
2575 LLVMPositionBuilderAtEnd (builder, entry_bb);
2578 ji = g_new0 (MonoJumpInfo, 1);
2579 ji->type = MONO_PATCH_INFO_AOT_MODULE;
2580 ji = mono_aot_patch_info_dup (ji);
2581 got_offset = mono_aot_get_got_offset (ji);
2582 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2583 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2584 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2585 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2586 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
2587 args [1] = LLVMGetParam (func, 0);
2589 args [2] = LLVMGetParam (func, 1);
2591 ji = g_new0 (MonoJumpInfo, 1);
2592 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
2593 ji->data.name = icall_name;
2594 ji = mono_aot_patch_info_dup (ji);
2595 got_offset = mono_aot_get_got_offset (ji);
2596 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2597 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2598 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2599 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2600 callee = LLVMBuildLoad (builder, got_entry_addr, "");
2601 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
2602 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
2604 // Set the inited flag
2605 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2606 indexes [1] = LLVMGetParam (func, 0);
2607 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
2609 LLVMBuildRetVoid (builder);
2611 LLVMVerifyFunction(func, LLVMAbortProcessAction);
2616 * Emit wrappers around the C icalls used to initialize llvm methods, to
2617 * make the calling code smaller and to enable usage of the llvm
2618 * PreserveAll calling convention.
2621 emit_init_icall_wrappers (MonoLLVMModule *module)
2623 module->init_method = emit_init_icall_wrapper (module, "init_method", "mono_aot_init_llvm_method", 0);
2624 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, "init_method_gshared_mrgctx", "mono_aot_init_gshared_method_mrgctx", 1);
2625 module->init_method_gshared_this = emit_init_icall_wrapper (module, "init_method_gshared_this", "mono_aot_init_gshared_method_this", 2);
2626 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, "init_method_gshared_vtable", "mono_aot_init_gshared_method_vtable", 3);
2630 emit_llvm_code_end (MonoLLVMModule *module)
2632 LLVMModuleRef lmodule = module->lmodule;
2634 LLVMBasicBlockRef entry_bb;
2635 LLVMBuilderRef builder;
2637 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2638 LLVMSetLinkage (func, LLVMInternalLinkage);
2639 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2640 module->code_end = func;
2641 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2642 builder = LLVMCreateBuilder ();
2643 LLVMPositionBuilderAtEnd (builder, entry_bb);
2644 LLVMBuildRetVoid (builder);
2648 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
2650 gboolean need_div_check = ctx->cfg->backend->need_div_check;
2653 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
2654 need_div_check = TRUE;
2656 if (!need_div_check)
2659 switch (ins->opcode) {
2672 case OP_IDIV_UN_IMM:
2673 case OP_LDIV_UN_IMM:
2674 case OP_IREM_UN_IMM:
2675 case OP_LREM_UN_IMM: {
2677 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
2678 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
2680 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
2681 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
2684 builder = ctx->builder;
2686 /* b == -1 && a == 0x80000000 */
2688 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
2689 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
2690 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
2692 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
2693 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
2696 builder = ctx->builder;
2708 * Emit code to initialize the GOT slots used by the method.
2711 emit_init_method (EmitContext *ctx)
2713 LLVMValueRef indexes [16], args [16], callee;
2714 LLVMValueRef inited_var, cmp, call;
2715 LLVMBasicBlockRef inited_bb, notinited_bb;
2716 LLVMBuilderRef builder = ctx->builder;
2717 MonoCompile *cfg = ctx->cfg;
2719 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
2721 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2722 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
2723 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
2725 args [0] = inited_var;
2726 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
2727 inited_var = LLVMBuildCall (ctx->builder, LLVMGetNamedFunction (ctx->lmodule, "llvm.expect.i8"), args, 2, "");
2729 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
2731 inited_bb = ctx->inited_bb;
2732 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
2734 LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
2736 builder = ctx->builder = create_builder (ctx);
2737 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
2740 if (ctx->rgctx_arg && cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) {
2741 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2742 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2743 callee = ctx->module->init_method_gshared_mrgctx;
2744 call = LLVMBuildCall (builder, callee, args, 2, "");
2745 } else if (ctx->rgctx_arg) {
2746 /* A vtable is passed as the rgctx argument */
2747 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2748 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2749 callee = ctx->module->init_method_gshared_vtable;
2750 call = LLVMBuildCall (builder, callee, args, 2, "");
2751 } else if (cfg->gshared) {
2752 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2753 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
2754 callee = ctx->module->init_method_gshared_this;
2755 call = LLVMBuildCall (builder, callee, args, 2, "");
2757 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2758 callee = ctx->module->init_method;
2759 call = LLVMBuildCall (builder, callee, args, 1, "");
2763 * This enables llvm to keep arguments in their original registers/
2764 * scratch registers, since the call will not clobber them.
2766 mono_llvm_set_call_preserveall_cc (call);
2768 LLVMBuildBr (builder, inited_bb);
2769 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
2771 builder = ctx->builder = create_builder (ctx);
2772 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
2776 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
2779 * Emit unbox trampoline using a tail call
2781 LLVMValueRef tramp, call, *args;
2782 LLVMBuilderRef builder;
2783 LLVMBasicBlockRef lbb;
2784 LLVMCallInfo *linfo;
2788 tramp_name = g_strdup_printf ("ut_%s", method_name);
2789 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
2790 LLVMSetLinkage (tramp, LLVMInternalLinkage);
2791 LLVMAddFunctionAttr (tramp, LLVMOptimizeForSizeAttribute);
2792 //LLVMAddFunctionAttr (tramp, LLVMNoUnwindAttribute);
2794 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
2795 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2796 LLVMAddAttribute (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVMInRegAttribute);
2797 if (ctx->cfg->vret_addr) {
2798 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
2799 if (linfo->ret.storage == LLVMArgVtypeByRef) {
2800 LLVMAddAttribute (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVMStructRetAttribute);
2801 LLVMAddAttribute (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
2805 lbb = LLVMAppendBasicBlock (tramp, "");
2806 builder = LLVMCreateBuilder ();
2807 LLVMPositionBuilderAtEnd (builder, lbb);
2809 nargs = LLVMCountParamTypes (method_type);
2810 args = g_new0 (LLVMValueRef, nargs);
2811 for (i = 0; i < nargs; ++i) {
2812 args [i] = LLVMGetParam (tramp, i);
2813 if (i == ctx->this_arg_pindex) {
2814 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
2816 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
2817 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), sizeof (MonoObject), FALSE), "");
2818 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
2821 call = LLVMBuildCall (builder, method, args, nargs, "");
2822 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2823 LLVMAddInstrAttribute (call, 1 + ctx->rgctx_arg_pindex, LLVMInRegAttribute);
2824 if (linfo->ret.storage == LLVMArgVtypeByRef)
2825 LLVMAddInstrAttribute (call, 1 + linfo->vret_arg_pindex, LLVMStructRetAttribute);
2827 // FIXME: This causes assertions in clang
2828 //mono_llvm_set_must_tail (call);
2829 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
2830 LLVMBuildRetVoid (builder);
2832 LLVMBuildRet (builder, call);
2834 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
2840 * Emit code to load/convert arguments.
2843 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
2846 MonoCompile *cfg = ctx->cfg;
2847 MonoMethodSignature *sig = ctx->sig;
2848 LLVMCallInfo *linfo = ctx->linfo;
2852 LLVMBuilderRef old_builder = ctx->builder;
2853 ctx->builder = builder;
2855 ctx->alloca_builder = create_builder (ctx);
2858 * Handle indirect/volatile variables by allocating memory for them
2859 * using 'alloca', and storing their address in a temporary.
2861 for (i = 0; i < cfg->num_varinfo; ++i) {
2862 MonoInst *var = cfg->varinfo [i];
2865 if (var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET) {
2866 } else if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
2867 vtype = type_to_llvm_type (ctx, var->inst_vtype);
2870 /* Could be already created by an OP_VPHI */
2871 if (!ctx->addresses [var->dreg]) {
2872 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
2873 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
2875 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
2879 names = g_new (char *, sig->param_count);
2880 mono_method_get_param_names (cfg->method, (const char **) names);
2882 for (i = 0; i < sig->param_count; ++i) {
2883 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
2884 int reg = cfg->args [i + sig->hasthis]->dreg;
2887 pindex = ainfo->pindex;
2889 switch (ainfo->storage) {
2890 case LLVMArgVtypeInReg:
2891 case LLVMArgAsFpArgs: {
2892 LLVMValueRef args [8];
2895 pindex += ainfo->ndummy_fpargs;
2897 /* The argument is received as a set of int/fp arguments, store them into the real argument */
2898 memset (args, 0, sizeof (args));
2899 if (ainfo->storage == LLVMArgVtypeInReg) {
2900 args [0] = LLVMGetParam (ctx->lmethod, pindex);
2901 if (ainfo->pair_storage [1] != LLVMArgNone)
2902 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
2904 g_assert (ainfo->nslots <= 8);
2905 for (j = 0; j < ainfo->nslots; ++j)
2906 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
2908 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2910 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
2912 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2913 /* Treat these as normal values */
2914 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2918 case LLVMArgVtypeByVal: {
2919 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2921 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2922 /* Treat these as normal values */
2923 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2927 case LLVMArgVtypeByRef: {
2928 /* The argument is passed by ref */
2929 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2932 case LLVMArgScalarByRef: {
2934 name = g_strdup_printf ("arg_%s", names [i]);
2936 name = g_strdup_printf ("arg_%d", i);
2937 ctx->values [reg] = LLVMBuildLoad (builder, LLVMGetParam (ctx->lmethod, pindex), name);
2941 case LLVMArgAsIArgs: {
2942 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2945 /* The argument is received as an array of ints, store it into the real argument */
2946 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2948 size = mono_class_value_size (mono_class_from_mono_type (ainfo->type), NULL);
2949 if (size < SIZEOF_VOID_P) {
2950 /* The upper bits of the registers might not be valid */
2951 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
2952 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
2953 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
2955 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
2959 case LLVMArgVtypeAsScalar:
2960 g_assert_not_reached ();
2962 case LLVMArgGsharedvtFixed: {
2963 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
2964 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2967 name = g_strdup_printf ("arg_%s", names [i]);
2969 name = g_strdup_printf ("arg_%d", i);
2971 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
2974 case LLVMArgGsharedvtFixedVtype: {
2975 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2978 name = g_strdup_printf ("vtype_arg_%s", names [i]);
2980 name = g_strdup_printf ("vtype_arg_%d", i);
2982 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
2983 g_assert (ctx->addresses [reg]);
2984 LLVMSetValueName (ctx->addresses [reg], name);
2985 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
2988 case LLVMArgGsharedvtVariable:
2989 /* The IR treats these as variables with addresses */
2990 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2993 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
3000 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3002 emit_volatile_store (ctx, cfg->args [0]->dreg);
3003 for (i = 0; i < sig->param_count; ++i)
3004 if (!mini_type_is_vtype (sig->params [i]))
3005 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3007 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3008 LLVMValueRef this_alloc;
3011 * The exception handling code needs the location where the this argument was
3012 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3013 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3014 * location into the LSDA.
3016 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3017 /* This volatile store will keep the alloca alive */
3018 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3020 set_metadata_flag (this_alloc, "mono.this");
3023 if (cfg->rgctx_var) {
3024 LLVMValueRef rgctx_alloc, store;
3027 * We handle the rgctx arg similarly to the this pointer.
3029 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3030 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3031 /* This volatile store will keep the alloca alive */
3032 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3034 set_metadata_flag (rgctx_alloc, "mono.this");
3037 /* Initialize the method if needed */
3038 if (cfg->compile_aot && ctx->llvm_only) {
3039 /* Emit a location for the initialization code */
3040 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3041 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3043 LLVMBuildBr (ctx->builder, ctx->init_bb);
3044 builder = ctx->builder = create_builder (ctx);
3045 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3046 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3049 /* Compute nesting between clauses */
3050 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3051 for (i = 0; i < cfg->header->num_clauses; ++i) {
3052 for (j = 0; j < cfg->header->num_clauses; ++j) {
3053 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3054 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3056 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3057 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3062 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3063 * it needs to continue normally, or return back to the exception handling system.
3065 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3069 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3072 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3073 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3074 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3076 if (bb->in_scount == 0) {
3079 sprintf (name, "finally_ind_bb%d", bb->block_num);
3080 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3081 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3083 ctx->bblocks [bb->block_num].finally_ind = val;
3085 /* Create a variable to hold the exception var */
3087 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3091 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3092 * LLVM bblock containing a landing pad causes problems for the
3093 * LLVM optimizer passes.
3095 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3096 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3098 ctx->builder = old_builder;
3102 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3104 MonoCompile *cfg = ctx->cfg;
3105 LLVMModuleRef lmodule = ctx->lmodule;
3106 LLVMValueRef *values = ctx->values;
3107 LLVMValueRef *addresses = ctx->addresses;
3108 MonoCallInst *call = (MonoCallInst*)ins;
3109 MonoMethodSignature *sig = call->signature;
3110 LLVMValueRef callee = NULL, lcall;
3112 LLVMCallInfo *cinfo;
3116 LLVMTypeRef llvm_sig;
3118 gboolean is_virtual, calli, preserveall;
3119 LLVMBuilderRef builder = *builder_ref;
3121 if (call->signature->call_convention != MONO_CALL_DEFAULT) {
3122 set_failure (ctx, "non-default callconv");
3126 cinfo = call->cinfo;
3128 if (call->rgctx_arg_reg)
3129 cinfo->rgctx_arg = TRUE;
3130 if (call->imt_arg_reg)
3131 cinfo->imt_arg = TRUE;
3133 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgScalarRetAddr || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3135 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3139 is_virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE || ins->opcode == OP_RCALL_MEMBASE);
3140 calli = !call->fptr_is_patch && (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG || ins->opcode == OP_RCALL_REG);
3142 preserveall = FALSE;
3144 /* FIXME: Avoid creating duplicate methods */
3146 if (ins->flags & MONO_INST_HAS_METHOD) {
3150 if (cfg->compile_aot) {
3151 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3153 set_failure (ctx, "can't encode patch");
3156 if (cfg->llvm_only && call->method->klass->image->assembly == ctx->module->assembly) {
3158 * Collect instructions representing the callee into a hash so they can be replaced
3159 * by the llvm method for the callee if the callee turns out to be direct
3160 * callable. Currently this only requires it to not fail llvm compilation.
3162 GSList *l = (GSList*)g_hash_table_lookup (ctx->method_to_callers, call->method);
3163 l = g_slist_prepend (l, callee);
3164 g_hash_table_insert (ctx->method_to_callers, call->method, l);
3167 callee = LLVMAddFunction (lmodule, "", llvm_sig);
3170 mono_create_jit_trampoline_in_domain (mono_domain_get (),
3172 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3176 if (!cfg->llvm_only && call->method && strstr (call->method->klass->name, "AsyncVoidMethodBuilder")) {
3177 /* LLVM miscompiles async methods */
3178 set_failure (ctx, "#13734");
3183 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
3189 memset (&ji, 0, sizeof (ji));
3190 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
3191 ji.data.target = info->name;
3193 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
3195 if (cfg->compile_aot) {
3196 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
3198 set_failure (ctx, "can't encode patch");
3202 callee = LLVMAddFunction (lmodule, "", llvm_sig);
3203 target = (gpointer)mono_icall_get_wrapper (info);
3204 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3207 if (cfg->compile_aot) {
3209 if (cfg->abs_patches) {
3210 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3212 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3214 set_failure (ctx, "can't encode patch");
3220 set_failure (ctx, "aot");
3224 callee = LLVMAddFunction (lmodule, "", llvm_sig);
3226 if (cfg->abs_patches) {
3227 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3232 * FIXME: Some trampolines might have
3233 * their own calling convention on some platforms.
3235 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, &error);
3236 mono_error_assert_ok (&error);
3237 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3241 LLVMAddGlobalMapping (ctx->module->ee, callee, (gpointer)call->fptr);
3247 int size = sizeof (gpointer);
3250 g_assert (ins->inst_offset % size == 0);
3251 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3253 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3255 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3257 if (ins->flags & MONO_INST_HAS_METHOD) {
3262 * Collect and convert arguments
3264 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg;
3265 len = sizeof (LLVMValueRef) * nargs;
3266 args = (LLVMValueRef*)alloca (len);
3267 memset (args, 0, len);
3268 l = call->out_ireg_args;
3270 if (call->rgctx_arg_reg) {
3271 g_assert (values [call->rgctx_arg_reg]);
3272 g_assert (cinfo->rgctx_arg_pindex < nargs);
3274 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3275 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3276 * it using a volatile load.
3279 if (!ctx->imt_rgctx_loc)
3280 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3281 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3282 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
3284 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3287 if (call->imt_arg_reg) {
3288 g_assert (!ctx->llvm_only);
3289 g_assert (values [call->imt_arg_reg]);
3290 g_assert (cinfo->imt_arg_pindex < nargs);
3292 if (!ctx->imt_rgctx_loc)
3293 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3294 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3295 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
3297 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3300 switch (cinfo->ret.storage) {
3301 case LLVMArgGsharedvtVariable: {
3302 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
3304 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
3305 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
3307 g_assert (addresses [call->inst.dreg]);
3308 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3314 if (!addresses [call->inst.dreg])
3315 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3316 g_assert (cinfo->vret_arg_pindex < nargs);
3317 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3318 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3320 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
3326 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
3327 * use the real callee for argument type conversion.
3329 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
3330 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
3331 LLVMGetParamTypes (callee_type, param_types);
3333 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3336 LLVMArgInfo *ainfo = &call->cinfo->args [i];
3338 pindex = ainfo->pindex;
3340 regpair = (guint32)(gssize)(l->data);
3341 reg = regpair & 0xffffff;
3342 args [pindex] = values [reg];
3343 switch (ainfo->storage) {
3344 case LLVMArgVtypeInReg:
3345 case LLVMArgAsFpArgs: {
3349 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
3350 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
3351 pindex += ainfo->ndummy_fpargs;
3353 g_assert (addresses [reg]);
3354 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
3358 // FIXME: Get rid of the VMOVE
3361 case LLVMArgVtypeByVal:
3362 g_assert (addresses [reg]);
3363 args [pindex] = addresses [reg];
3365 case LLVMArgVtypeByRef:
3366 case LLVMArgScalarByRef: {
3367 g_assert (addresses [reg]);
3368 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3371 case LLVMArgAsIArgs:
3372 g_assert (addresses [reg]);
3373 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
3375 case LLVMArgVtypeAsScalar:
3376 g_assert_not_reached ();
3378 case LLVMArgGsharedvtFixed:
3379 case LLVMArgGsharedvtFixedVtype:
3380 g_assert (addresses [reg]);
3381 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3383 case LLVMArgGsharedvtVariable:
3384 g_assert (addresses [reg]);
3385 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
3388 g_assert (args [pindex]);
3389 if (i == 0 && sig->hasthis)
3390 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
3392 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
3395 g_assert (pindex <= nargs);
3400 // FIXME: Align call sites
3406 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
3409 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
3411 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
3412 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
3414 /* The two can't be used together, so use only one LLVM calling conv to pass them */
3415 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
3416 if (!sig->pinvoke && !cfg->llvm_only)
3417 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
3419 mono_llvm_set_call_preserveall_cc (lcall);
3421 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3422 LLVMAddInstrAttribute (lcall, 1 + cinfo->vret_arg_pindex, LLVMStructRetAttribute);
3423 if (!ctx->llvm_only && call->rgctx_arg_reg)
3424 LLVMAddInstrAttribute (lcall, 1 + cinfo->rgctx_arg_pindex, LLVMInRegAttribute);
3425 if (call->imt_arg_reg)
3426 LLVMAddInstrAttribute (lcall, 1 + cinfo->imt_arg_pindex, LLVMInRegAttribute);
3428 /* Add byval attributes if needed */
3429 for (i = 0; i < sig->param_count; ++i) {
3430 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
3432 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
3433 LLVMAddInstrAttribute (lcall, 1 + ainfo->pindex, LLVMByValAttribute);
3437 * Convert the result
3439 switch (cinfo->ret.storage) {
3440 case LLVMArgVtypeInReg: {
3441 LLVMValueRef regs [2];
3443 if (LLVMTypeOf (lcall) == LLVMVoidType ())
3447 if (!addresses [ins->dreg])
3448 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
3450 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
3451 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
3452 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
3453 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
3456 case LLVMArgVtypeByVal:
3457 if (!addresses [call->inst.dreg])
3458 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3459 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
3461 case LLVMArgFpStruct:
3462 if (!addresses [call->inst.dreg])
3463 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3464 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3466 case LLVMArgVtypeAsScalar:
3467 if (!addresses [call->inst.dreg])
3468 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3469 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3471 case LLVMArgVtypeRetAddr:
3472 case LLVMArgVtypeByRef:
3473 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret))) {
3474 /* Some opcodes like STOREX_MEMBASE access these by value */
3475 g_assert (addresses [call->inst.dreg]);
3476 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3479 case LLVMArgScalarRetAddr:
3480 /* Normal scalar returned using a vtype return argument */
3481 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3483 case LLVMArgGsharedvtVariable:
3485 case LLVMArgGsharedvtFixed:
3486 case LLVMArgGsharedvtFixedVtype:
3487 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3490 if (sig->ret->type != MONO_TYPE_VOID)
3491 /* If the method returns an unsigned value, need to zext it */
3492 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
3496 *builder_ref = ctx->builder;
3500 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3502 const char *icall_name = rethrow ? "mono_llvm_rethrow_exception" : "mono_llvm_throw_exception";
3503 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3505 LLVMTypeRef exc_type = type_to_llvm_type (ctx, &mono_get_exception_class ()->byval_arg);
3508 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
3510 if (ctx->cfg->compile_aot) {
3511 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, icall_name);
3513 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3514 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3515 mono_memory_barrier ();
3518 ctx->module->rethrow = callee;
3520 ctx->module->throw_icall = callee;
3524 LLVMValueRef args [2];
3526 args [0] = convert (ctx, exc, exc_type);
3527 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
3529 LLVMBuildUnreachable (ctx->builder);
3531 ctx->builder = create_builder (ctx);
3535 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3537 MonoMethodSignature *throw_sig;
3538 LLVMValueRef callee, arg;
3539 const char *icall_name;
3541 callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3542 icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
3545 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
3546 throw_sig->ret = &mono_get_void_class ()->byval_arg;
3547 throw_sig->params [0] = &mono_get_object_class ()->byval_arg;
3548 if (ctx->cfg->compile_aot) {
3549 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3551 callee = LLVMAddFunction (ctx->lmodule, icall_name, sig_to_llvm_sig (ctx, throw_sig));
3555 * LLVM doesn't push the exception argument, so we need a different
3558 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline"));
3560 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3564 mono_memory_barrier ();
3566 ctx->module->rethrow = callee;
3568 ctx->module->throw_icall = callee;
3570 arg = convert (ctx, exc, type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg));
3571 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
3575 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
3577 const char *icall_name = "mono_llvm_resume_exception";
3578 LLVMValueRef callee = ctx->module->resume_eh;
3580 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3583 if (ctx->cfg->compile_aot) {
3584 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3586 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3587 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3588 mono_memory_barrier ();
3590 ctx->module->resume_eh = callee;
3594 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
3596 LLVMBuildUnreachable (ctx->builder);
3598 ctx->builder = create_builder (ctx);
3602 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3604 const char *icall_name = "mono_llvm_clear_exception";
3606 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3607 LLVMValueRef callee = NULL;
3610 if (ctx->cfg->compile_aot) {
3611 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3613 // FIXME: This is broken.
3614 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3618 g_assert (builder && callee);
3620 return LLVMBuildCall (builder, callee, NULL, 0, "");
3624 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3626 const char *icall_name = "mono_llvm_load_exception";
3628 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
3629 LLVMValueRef callee = NULL;
3632 if (ctx->cfg->compile_aot) {
3633 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3635 // FIXME: This is broken.
3636 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3640 g_assert (builder && callee);
3642 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
3647 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
3649 const char *icall_name = "mono_llvm_match_exception";
3651 ctx->builder = builder;
3653 const int num_args = 5;
3654 LLVMValueRef args [num_args];
3655 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
3656 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
3657 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
3658 if (ctx->cfg->rgctx_var) {
3659 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
3660 g_assert (rgctx_alloc);
3661 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
3663 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
3666 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
3668 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
3670 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
3671 LLVMValueRef callee = ctx->module->match_exc;
3674 if (ctx->cfg->compile_aot) {
3675 ctx->builder = builder;
3676 // get_callee expects ctx->builder to be the emitting builder
3677 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3679 callee = ctx->module->match_exc = LLVMAddFunction (ctx->lmodule, icall_name, match_sig);
3680 LLVMAddGlobalMapping (ctx->module->ee, ctx->module->match_exc, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3681 ctx->module->match_exc = callee;
3682 mono_memory_barrier ();
3686 g_assert (builder && callee);
3688 g_assert (ctx->ex_var);
3690 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
3693 // FIXME: This won't work because the code-finding makes this
3695 /*#define MONO_PERSONALITY_DEBUG*/
3697 #ifdef MONO_PERSONALITY_DEBUG
3698 static const gboolean use_debug_personality = TRUE;
3699 static const char *default_personality_name = "mono_debug_personality";
3701 static const gboolean use_debug_personality = FALSE;
3702 static const char *default_personality_name = "__gxx_personality_v0";
3706 default_cpp_lpad_exc_signature (void)
3708 static gboolean inited = FALSE;
3709 static LLVMTypeRef sig;
3712 LLVMTypeRef signature [2];
3713 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
3714 signature [1] = LLVMInt32Type ();
3715 sig = LLVMStructType (signature, 2, FALSE);
3723 get_mono_personality (EmitContext *ctx)
3725 LLVMValueRef personality = NULL;
3726 static gint32 mapping_inited = FALSE;
3727 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
3729 if (!use_debug_personality) {
3730 if (ctx->cfg->compile_aot) {
3731 personality = LLVMGetNamedFunction (ctx->lmodule, default_personality_name);
3732 } else if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0) {
3733 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3734 LLVMAddGlobalMapping (ctx->module->ee, personality, personality);
3737 if (ctx->cfg->compile_aot) {
3738 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name);
3740 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3741 LLVMAddGlobalMapping (ctx->module->ee, personality, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name));
3742 mono_memory_barrier ();
3746 g_assert (personality);
3750 static LLVMBasicBlockRef
3751 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
3753 MonoCompile *cfg = ctx->cfg;
3754 LLVMBuilderRef old_builder = ctx->builder;
3755 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
3757 LLVMBuilderRef lpadBuilder = create_builder (ctx);
3758 ctx->builder = lpadBuilder;
3760 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
3761 g_assert (handler_bb);
3763 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3764 LLVMValueRef personality = get_mono_personality (ctx);
3765 g_assert (personality);
3767 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
3768 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
3770 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3771 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
3772 g_assert (landing_pad);
3774 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
3775 LLVMAddClause (landing_pad, cast);
3777 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
3778 LLVMBuilderRef resume_builder = create_builder (ctx);
3779 ctx->builder = resume_builder;
3780 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
3782 emit_resume_eh (ctx, handler_bb);
3785 ctx->builder = lpadBuilder;
3786 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3788 gboolean finally_only = TRUE;
3790 MonoExceptionClause *group_cursor = group_start;
3792 for (int i = 0; i < group_size; i ++) {
3793 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY))
3794 finally_only = FALSE;
3800 // Handle landing pad inlining
3802 if (!finally_only) {
3803 // So at each level of the exception stack we will match the exception again.
3804 // During that match, we need to compare against the handler types for the current
3805 // protected region. We send the try start and end so that we can only check against
3806 // handlers for this lexical protected region.
3807 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
3809 // if returns -1, resume
3810 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
3812 // else move to that target bb
3813 for (int i=0; i < group_size; i++) {
3814 MonoExceptionClause *clause = group_start + i;
3815 int clause_index = clause - cfg->header->clauses;
3816 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3817 g_assert (handler_bb);
3818 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3819 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3822 int clause_index = group_start - cfg->header->clauses;
3823 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3824 g_assert (finally_bb);
3826 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
3829 ctx->builder = old_builder;
3836 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
3838 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3839 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
3841 // Make exception available to catch blocks
3842 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY)) {
3843 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
3845 g_assert (ctx->ex_var);
3846 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
3848 if (bb->in_scount == 1) {
3849 MonoInst *exvar = bb->in_stack [0];
3850 g_assert (!ctx->values [exvar->dreg]);
3851 g_assert (ctx->ex_var);
3852 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
3853 emit_volatile_store (ctx, exvar->dreg);
3856 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
3859 LLVMBuilderRef handler_builder = create_builder (ctx);
3860 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
3861 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
3863 // Make the handler code end with a jump to cbb
3864 LLVMBuildBr (handler_builder, cbb);
3868 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
3870 MonoCompile *cfg = ctx->cfg;
3871 LLVMValueRef *values = ctx->values;
3872 LLVMModuleRef lmodule = ctx->lmodule;
3873 BBInfo *bblocks = ctx->bblocks;
3875 LLVMValueRef personality;
3876 LLVMValueRef landing_pad;
3877 LLVMBasicBlockRef target_bb;
3879 static gint32 mapping_inited;
3880 static int ti_generator;
3883 LLVMValueRef type_info;
3887 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3889 if (cfg->compile_aot) {
3890 /* Use a dummy personality function */
3891 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3892 g_assert (personality);
3894 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3895 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
3896 LLVMAddGlobalMapping (ctx->module->ee, personality, (gpointer)mono_personality);
3899 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
3901 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
3904 * Create the type info
3906 sprintf (ti_name, "type_info_%d", ti_generator);
3909 if (cfg->compile_aot) {
3910 /* decode_eh_frame () in aot-runtime.c will decode this */
3911 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
3912 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
3915 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
3917 LLVMSetLinkage (type_info, LLVMInternalLinkage);
3920 * After the cfg mempool is freed, the type info will point to stale memory,
3921 * but this is not a problem, since we decode it once in exception_cb during
3924 ti = (gint32*)mono_mempool_alloc (cfg->mempool, sizeof (gint32));
3925 *(gint32*)ti = clause_index;
3927 type_info = LLVMAddGlobal (lmodule, i8ptr, ti_name);
3929 LLVMAddGlobalMapping (ctx->module->ee, type_info, ti);
3933 LLVMTypeRef members [2], ret_type;
3935 members [0] = i8ptr;
3936 members [1] = LLVMInt32Type ();
3937 ret_type = LLVMStructType (members, 2, FALSE);
3939 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
3940 LLVMAddClause (landing_pad, type_info);
3942 /* Store the exception into the exvar */
3944 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
3948 * LLVM throw sites are associated with a one landing pad, and LLVM generated
3949 * code expects control to be transferred to this landing pad even in the
3950 * presence of nested clauses. The landing pad needs to branch to the landing
3951 * pads belonging to nested clauses based on the selector value returned by
3952 * the landing pad instruction, which is passed to the landing pad in a
3953 * register by the EH code.
3955 target_bb = bblocks [bb->block_num].call_handler_target_bb;
3956 g_assert (target_bb);
3959 * Branch to the correct landing pad
3961 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
3962 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
3964 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
3965 int nesting_clause_index = GPOINTER_TO_INT (l->data);
3966 MonoBasicBlock *handler_bb;
3968 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
3969 g_assert (handler_bb);
3971 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3972 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3975 /* Start a new bblock which CALL_HANDLER can branch to */
3976 target_bb = bblocks [bb->block_num].call_handler_target_bb;
3978 ctx->builder = builder = create_builder (ctx);
3979 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
3981 ctx->bblocks [bb->block_num].end_bblock = target_bb;
3983 /* Store the exception into the IL level exvar */
3984 if (bb->in_scount == 1) {
3985 g_assert (bb->in_scount == 1);
3986 exvar = bb->in_stack [0];
3988 // FIXME: This is shared with filter clauses ?
3989 g_assert (!values [exvar->dreg]);
3991 g_assert (ctx->ex_var);
3992 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
3993 emit_volatile_store (ctx, exvar->dreg);
3999 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4001 MonoCompile *cfg = ctx->cfg;
4002 MonoMethodSignature *sig = ctx->sig;
4003 LLVMValueRef method = ctx->lmethod;
4004 LLVMValueRef *values = ctx->values;
4005 LLVMValueRef *addresses = ctx->addresses;
4006 LLVMCallInfo *linfo = ctx->linfo;
4007 LLVMModuleRef lmodule = ctx->lmodule;
4008 BBInfo *bblocks = ctx->bblocks;
4010 LLVMBasicBlockRef cbb;
4011 LLVMBuilderRef builder, starting_builder;
4012 gboolean has_terminator;
4014 LLVMValueRef lhs, rhs;
4017 cbb = get_end_bb (ctx, bb);
4019 builder = create_builder (ctx);
4020 ctx->builder = builder;
4021 LLVMPositionBuilderAtEnd (builder, cbb);
4026 if (bb->flags & BB_EXCEPTION_HANDLER) {
4027 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4028 set_failure (ctx, "handler without invokes");
4033 emit_llvmonly_handler_start (ctx, bb, cbb);
4035 emit_handler_start (ctx, bb, builder);
4038 builder = ctx->builder;
4041 has_terminator = FALSE;
4042 starting_builder = builder;
4043 for (ins = bb->code; ins; ins = ins->next) {
4044 const char *spec = LLVM_INS_INFO (ins->opcode);
4046 char dname_buf [128];
4048 emit_dbg_loc (ctx, builder, ins->cil_code);
4053 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4054 * Start a new bblock. If the llvm optimization passes merge these, we
4055 * can work around that by doing a volatile load + cond branch from
4056 * localloc-ed memory.
4058 //set_failure (ctx, "basic block too long");
4059 cbb = gen_bb (ctx, "CONT_LONG_BB");
4060 LLVMBuildBr (ctx->builder, cbb);
4061 ctx->builder = builder = create_builder (ctx);
4062 LLVMPositionBuilderAtEnd (builder, cbb);
4063 ctx->bblocks [bb->block_num].end_bblock = cbb;
4068 /* There could be instructions after a terminator, skip them */
4071 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4072 sprintf (dname_buf, "t%d", ins->dreg);
4076 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4077 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4079 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4080 lhs = emit_volatile_load (ctx, ins->sreg1);
4082 /* It is ok for SETRET to have an uninitialized argument */
4083 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4084 set_failure (ctx, "sreg1");
4087 lhs = values [ins->sreg1];
4093 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4094 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4095 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4096 rhs = emit_volatile_load (ctx, ins->sreg2);
4098 if (!values [ins->sreg2]) {
4099 set_failure (ctx, "sreg2");
4102 rhs = values [ins->sreg2];
4108 //mono_print_ins (ins);
4109 switch (ins->opcode) {
4112 case OP_LIVERANGE_START:
4113 case OP_LIVERANGE_END:
4116 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4119 #if SIZEOF_VOID_P == 4
4120 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4122 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4126 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
4130 values [ins->dreg] = LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0);
4132 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
4134 case OP_DUMMY_ICONST:
4135 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4137 case OP_DUMMY_I8CONST:
4138 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4140 case OP_DUMMY_R8CONST:
4141 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4144 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4145 LLVMBuildBr (builder, target_bb);
4146 has_terminator = TRUE;
4153 LLVMBasicBlockRef new_bb;
4154 LLVMBuilderRef new_builder;
4156 // The default branch is already handled
4157 // FIXME: Handle it here
4159 /* Start new bblock */
4160 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4161 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4163 lhs = convert (ctx, lhs, LLVMInt32Type ());
4164 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4165 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4166 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4168 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4171 new_builder = create_builder (ctx);
4172 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4173 LLVMBuildUnreachable (new_builder);
4175 has_terminator = TRUE;
4176 g_assert (!ins->next);
4182 switch (linfo->ret.storage) {
4183 case LLVMArgVtypeInReg: {
4184 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4185 LLVMValueRef val, addr, retval;
4188 retval = LLVMGetUndef (ret_type);
4190 if (!addresses [ins->sreg1]) {
4192 * The return type is an LLVM vector type, have to convert between it and the
4193 * real return type which is a struct type.
4195 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret)));
4196 /* Convert to 2xi64 first */
4197 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4199 for (i = 0; i < 2; ++i) {
4200 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4201 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
4203 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4207 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
4208 for (i = 0; i < 2; ++i) {
4209 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4210 LLVMValueRef indexes [2], part_addr;
4212 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4213 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
4214 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
4216 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
4218 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4222 LLVMBuildRet (builder, retval);
4225 case LLVMArgVtypeAsScalar: {
4226 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4227 LLVMValueRef retval;
4229 g_assert (addresses [ins->sreg1]);
4231 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
4232 LLVMBuildRet (builder, retval);
4235 case LLVMArgVtypeByVal: {
4236 LLVMValueRef retval;
4238 g_assert (addresses [ins->sreg1]);
4239 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
4240 LLVMBuildRet (builder, retval);
4243 case LLVMArgVtypeByRef: {
4244 LLVMBuildRetVoid (builder);
4247 case LLVMArgGsharedvtFixed: {
4248 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
4249 /* The return value is in lhs, need to store to the vret argument */
4250 /* sreg1 might not be set */
4252 g_assert (cfg->vret_addr);
4253 g_assert (values [cfg->vret_addr->dreg]);
4254 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
4256 LLVMBuildRetVoid (builder);
4259 case LLVMArgGsharedvtFixedVtype: {
4261 LLVMBuildRetVoid (builder);
4264 case LLVMArgGsharedvtVariable: {
4266 LLVMBuildRetVoid (builder);
4269 case LLVMArgVtypeRetAddr: {
4270 LLVMBuildRetVoid (builder);
4273 case LLVMArgScalarRetAddr: {
4274 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
4275 LLVMValueRef param = LLVMGetParam (method, linfo->vret_arg_pindex);
4277 /* sreg1 might not be set */
4279 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, param, LLVMPointerType (ret_type, 0)));
4280 LLVMBuildRetVoid (builder);
4283 case LLVMArgFpStruct: {
4284 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4285 LLVMValueRef retval;
4287 g_assert (addresses [ins->sreg1]);
4288 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
4289 LLVMBuildRet (builder, retval);
4293 case LLVMArgNormal: {
4294 if (!lhs || ctx->is_dead [ins->sreg1]) {
4296 * The method did not set its return value, probably because it
4297 * ends with a throw.
4300 LLVMBuildRetVoid (builder);
4302 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
4304 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
4306 has_terminator = TRUE;
4310 g_assert_not_reached ();
4319 case OP_ICOMPARE_IMM:
4320 case OP_LCOMPARE_IMM:
4321 case OP_COMPARE_IMM: {
4323 LLVMValueRef cmp, args [16];
4324 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
4326 if (ins->next->opcode == OP_NOP)
4329 if (ins->next->opcode == OP_BR)
4330 /* The comparison result is not needed */
4333 rel = mono_opcode_to_cond (ins->next->opcode);
4335 if (ins->opcode == OP_ICOMPARE_IMM) {
4336 lhs = convert (ctx, lhs, LLVMInt32Type ());
4337 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4339 if (ins->opcode == OP_LCOMPARE_IMM) {
4340 lhs = convert (ctx, lhs, LLVMInt64Type ());
4341 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4343 if (ins->opcode == OP_LCOMPARE) {
4344 lhs = convert (ctx, lhs, LLVMInt64Type ());
4345 rhs = convert (ctx, rhs, LLVMInt64Type ());
4347 if (ins->opcode == OP_ICOMPARE) {
4348 lhs = convert (ctx, lhs, LLVMInt32Type ());
4349 rhs = convert (ctx, rhs, LLVMInt32Type ());
4353 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4354 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
4355 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
4356 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
4359 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
4360 if (ins->opcode == OP_FCOMPARE) {
4361 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4362 } else if (ins->opcode == OP_RCOMPARE) {
4363 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4364 } else if (ins->opcode == OP_COMPARE_IMM) {
4365 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0)
4366 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
4368 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
4369 } else if (ins->opcode == OP_LCOMPARE_IMM) {
4370 if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) {
4371 /* The immediate is encoded in two fields */
4372 guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm);
4373 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), "");
4375 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), "");
4378 else if (ins->opcode == OP_COMPARE) {
4379 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
4380 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4382 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
4384 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4388 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
4389 cmp = LLVMBuildCall (ctx->builder, LLVMGetNamedFunction (ctx->lmodule, "llvm.expect.i1"), args, 2, "");
4392 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
4393 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
4395 * If the target bb contains PHI instructions, LLVM requires
4396 * two PHI entries for this bblock, while we only generate one.
4397 * So convert this to an unconditional bblock. (bxc #171).
4399 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
4401 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
4403 has_terminator = TRUE;
4404 } else if (MONO_IS_SETCC (ins->next)) {
4405 sprintf (dname_buf, "t%d", ins->next->dreg);
4407 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4409 /* Add stores for volatile variables */
4410 emit_volatile_store (ctx, ins->next->dreg);
4411 } else if (MONO_IS_COND_EXC (ins->next)) {
4412 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
4415 builder = ctx->builder;
4417 set_failure (ctx, "next");
4435 rel = mono_opcode_to_cond (ins->opcode);
4437 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4438 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4449 rel = mono_opcode_to_cond (ins->opcode);
4451 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4452 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4460 gboolean empty = TRUE;
4462 /* Check that all input bblocks really branch to us */
4463 for (i = 0; i < bb->in_count; ++i) {
4464 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4465 ins->inst_phi_args [i + 1] = -1;
4471 /* LLVM doesn't like phi instructions with zero operands */
4472 ctx->is_dead [ins->dreg] = TRUE;
4476 /* Created earlier, insert it now */
4477 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4479 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4480 int sreg1 = ins->inst_phi_args [i + 1];
4484 * Count the number of times the incoming bblock branches to us,
4485 * since llvm requires a separate entry for each.
4487 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4488 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4491 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4492 if (switch_ins->inst_many_bb [j] == bb)
4499 /* Remember for later */
4500 for (j = 0; j < count; ++j) {
4501 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4504 node->in_bb = bb->in_bb [i];
4506 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4516 values [ins->dreg] = lhs;
4520 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
4523 values [ins->dreg] = lhs;
4525 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
4527 * This is added by the spilling pass in case of the JIT,
4528 * but we have to do it ourselves.
4530 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
4534 case OP_MOVE_F_TO_I4: {
4535 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
4538 case OP_MOVE_I4_TO_F: {
4539 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
4542 case OP_MOVE_F_TO_I8: {
4543 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
4546 case OP_MOVE_I8_TO_F: {
4547 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
4580 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4581 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4583 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
4586 builder = ctx->builder;
4588 switch (ins->opcode) {
4591 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
4595 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
4599 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
4603 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
4607 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
4611 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
4615 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
4619 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4623 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
4627 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
4631 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
4635 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
4639 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
4643 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
4647 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4650 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4653 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4657 g_assert_not_reached ();
4664 lhs = convert (ctx, lhs, LLVMFloatType ());
4665 rhs = convert (ctx, rhs, LLVMFloatType ());
4666 switch (ins->opcode) {
4668 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4671 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4674 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4677 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4680 g_assert_not_reached ();
4689 case OP_IREM_UN_IMM:
4691 case OP_IDIV_UN_IMM:
4697 case OP_ISHR_UN_IMM:
4707 case OP_LSHR_UN_IMM:
4713 case OP_SHR_UN_IMM: {
4716 if (spec [MONO_INST_SRC1] == 'l') {
4717 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4719 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4722 emit_div_check (ctx, builder, bb, ins, lhs, imm);
4725 builder = ctx->builder;
4727 #if SIZEOF_VOID_P == 4
4728 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
4729 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4732 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4733 lhs = convert (ctx, lhs, IntPtrType ());
4734 imm = convert (ctx, imm, LLVMTypeOf (lhs));
4735 switch (ins->opcode) {
4739 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
4743 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
4748 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
4752 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
4754 case OP_IDIV_UN_IMM:
4755 case OP_LDIV_UN_IMM:
4756 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
4760 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
4762 case OP_IREM_UN_IMM:
4763 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
4768 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
4772 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
4776 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
4781 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
4786 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
4788 case OP_ISHR_UN_IMM:
4789 /* This is used to implement conv.u4, so the lhs could be an i8 */
4790 lhs = convert (ctx, lhs, LLVMInt32Type ());
4791 imm = convert (ctx, imm, LLVMInt32Type ());
4792 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4794 case OP_LSHR_UN_IMM:
4796 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4799 g_assert_not_reached ();
4804 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4807 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
4810 lhs = convert (ctx, lhs, LLVMDoubleType ());
4811 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
4814 lhs = convert (ctx, lhs, LLVMFloatType ());
4815 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
4818 guint32 v = 0xffffffff;
4819 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4823 guint64 v = 0xffffffffffffffffLL;
4824 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
4827 #if defined(TARGET_X86) || defined(TARGET_AMD64)
4829 LLVMValueRef v1, v2;
4831 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
4832 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
4833 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
4838 case OP_ICONV_TO_I1:
4839 case OP_ICONV_TO_I2:
4840 case OP_ICONV_TO_I4:
4841 case OP_ICONV_TO_U1:
4842 case OP_ICONV_TO_U2:
4843 case OP_ICONV_TO_U4:
4844 case OP_LCONV_TO_I1:
4845 case OP_LCONV_TO_I2:
4846 case OP_LCONV_TO_U1:
4847 case OP_LCONV_TO_U2:
4848 case OP_LCONV_TO_U4: {
4851 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
4853 /* Have to do two casts since our vregs have type int */
4854 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
4856 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
4858 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
4861 case OP_ICONV_TO_I8:
4862 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
4864 case OP_ICONV_TO_U8:
4865 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
4867 case OP_FCONV_TO_I4:
4868 case OP_RCONV_TO_I4:
4869 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
4871 case OP_FCONV_TO_I1:
4872 case OP_RCONV_TO_I1:
4873 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
4875 case OP_FCONV_TO_U1:
4876 case OP_RCONV_TO_U1:
4877 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
4879 case OP_FCONV_TO_I2:
4880 case OP_RCONV_TO_I2:
4881 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4883 case OP_FCONV_TO_U2:
4884 case OP_RCONV_TO_U2:
4885 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4887 case OP_RCONV_TO_U4:
4888 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
4890 case OP_FCONV_TO_I8:
4891 case OP_RCONV_TO_I8:
4892 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
4895 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
4897 case OP_ICONV_TO_R8:
4898 case OP_LCONV_TO_R8:
4899 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
4901 case OP_ICONV_TO_R_UN:
4902 case OP_LCONV_TO_R_UN:
4903 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
4905 #if SIZEOF_VOID_P == 4
4908 case OP_LCONV_TO_I4:
4909 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
4911 case OP_ICONV_TO_R4:
4912 case OP_LCONV_TO_R4:
4913 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
4915 values [ins->dreg] = v;
4917 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
4919 case OP_FCONV_TO_R4:
4920 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
4922 values [ins->dreg] = v;
4924 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
4926 case OP_RCONV_TO_R8:
4927 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
4929 case OP_RCONV_TO_R4:
4930 values [ins->dreg] = lhs;
4933 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
4936 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
4939 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
4941 case OP_LOCALLOC_IMM: {
4944 guint32 size = ins->inst_imm;
4945 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
4947 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
4949 if (ins->flags & MONO_INST_INIT) {
4950 LLVMValueRef args [5];
4953 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
4954 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
4955 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
4956 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
4957 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memset_func_name), args, memset_param_count, "");
4960 values [ins->dreg] = v;
4964 LLVMValueRef v, size;
4966 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
4968 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
4970 if (ins->flags & MONO_INST_INIT) {
4971 LLVMValueRef args [5];
4974 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
4976 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
4977 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
4978 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memset_func_name), args, memset_param_count, "");
4980 values [ins->dreg] = v;
4984 case OP_LOADI1_MEMBASE:
4985 case OP_LOADU1_MEMBASE:
4986 case OP_LOADI2_MEMBASE:
4987 case OP_LOADU2_MEMBASE:
4988 case OP_LOADI4_MEMBASE:
4989 case OP_LOADU4_MEMBASE:
4990 case OP_LOADI8_MEMBASE:
4991 case OP_LOADR4_MEMBASE:
4992 case OP_LOADR8_MEMBASE:
4993 case OP_LOAD_MEMBASE:
5001 LLVMValueRef base, index, addr;
5003 gboolean sext = FALSE, zext = FALSE;
5004 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5006 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5011 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5012 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5017 if (ins->inst_offset == 0) {
5019 } else if (ins->inst_offset % size != 0) {
5020 /* Unaligned load */
5021 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5022 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5024 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5025 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5029 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5031 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, dname, is_volatile);
5033 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5035 * These will signal LLVM that these loads do not alias any stores, and
5036 * they can't fail, allowing them to be hoisted out of loops.
5038 set_invariant_load_flag (values [ins->dreg]);
5039 set_metadata_flag (values [ins->dreg], "mono.nofail.load");
5043 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5045 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5046 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5047 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5051 case OP_STOREI1_MEMBASE_REG:
5052 case OP_STOREI2_MEMBASE_REG:
5053 case OP_STOREI4_MEMBASE_REG:
5054 case OP_STOREI8_MEMBASE_REG:
5055 case OP_STORER4_MEMBASE_REG:
5056 case OP_STORER8_MEMBASE_REG:
5057 case OP_STORE_MEMBASE_REG: {
5059 LLVMValueRef index, addr;
5061 gboolean sext = FALSE, zext = FALSE;
5062 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5064 if (!values [ins->inst_destbasereg]) {
5065 set_failure (ctx, "inst_destbasereg");
5069 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5071 if (ins->inst_offset % size != 0) {
5072 /* Unaligned store */
5073 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5074 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5076 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5077 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5079 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
5083 case OP_STOREI1_MEMBASE_IMM:
5084 case OP_STOREI2_MEMBASE_IMM:
5085 case OP_STOREI4_MEMBASE_IMM:
5086 case OP_STOREI8_MEMBASE_IMM:
5087 case OP_STORE_MEMBASE_IMM: {
5089 LLVMValueRef index, addr;
5091 gboolean sext = FALSE, zext = FALSE;
5092 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5094 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5096 if (ins->inst_offset % size != 0) {
5097 /* Unaligned store */
5098 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5099 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5101 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5102 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5104 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
5109 emit_load (ctx, bb, &builder, sizeof (gpointer), convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
5111 case OP_OUTARG_VTRETADDR:
5119 case OP_VOIDCALL_MEMBASE:
5120 case OP_CALL_MEMBASE:
5121 case OP_LCALL_MEMBASE:
5122 case OP_FCALL_MEMBASE:
5123 case OP_RCALL_MEMBASE:
5124 case OP_VCALL_MEMBASE:
5125 case OP_VOIDCALL_REG:
5130 case OP_VCALL_REG: {
5131 process_call (ctx, bb, &builder, ins);
5136 LLVMValueRef indexes [2];
5137 MonoJumpInfo *tmp_ji, *ji;
5138 LLVMValueRef got_entry_addr;
5142 * FIXME: Can't allocate from the cfg mempool since that is freed if
5143 * the LLVM compile fails.
5145 tmp_ji = g_new0 (MonoJumpInfo, 1);
5146 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5147 tmp_ji->data.target = ins->inst_p0;
5149 ji = mono_aot_patch_info_dup (tmp_ji);
5152 ji->next = cfg->patch_info;
5153 cfg->patch_info = ji;
5155 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5156 got_offset = mono_aot_get_got_offset (cfg->patch_info);
5157 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5158 if (!mono_aot_is_shared_got_offset (got_offset)) {
5159 //mono_print_ji (ji);
5161 ctx->has_got_access = TRUE;
5164 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5165 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5166 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5168 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5169 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5171 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5172 if (!cfg->llvm_only)
5173 set_invariant_load_flag (values [ins->dreg]);
5176 case OP_NOT_REACHED:
5177 LLVMBuildUnreachable (builder);
5178 has_terminator = TRUE;
5179 g_assert (bb->block_num < cfg->max_block_num);
5180 ctx->unreachable [bb->block_num] = TRUE;
5181 /* Might have instructions after this */
5183 MonoInst *next = ins->next;
5185 * FIXME: If later code uses the regs defined by these instructions,
5186 * compilation will fail.
5188 MONO_DELETE_INS (bb, next);
5192 MonoInst *var = ins->inst_i0;
5194 if (var->opcode == OP_VTARG_ADDR) {
5195 /* The variable contains the vtype address */
5196 values [ins->dreg] = values [var->dreg];
5197 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
5198 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
5200 values [ins->dreg] = addresses [var->dreg];
5205 LLVMValueRef args [1];
5207 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5208 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "llvm.sin.f64"), args, 1, dname);
5212 LLVMValueRef args [1];
5214 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5215 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "llvm.cos.f64"), args, 1, dname);
5219 LLVMValueRef args [1];
5221 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5222 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "llvm.sqrt.f64"), args, 1, dname);
5226 LLVMValueRef args [1];
5228 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5229 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, "fabs"), args, 1, dname);
5243 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5244 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5246 switch (ins->opcode) {
5249 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
5253 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
5257 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
5261 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
5264 g_assert_not_reached ();
5267 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
5270 case OP_ATOMIC_EXCHANGE_I4:
5271 case OP_ATOMIC_EXCHANGE_I8: {
5272 LLVMValueRef args [2];
5275 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
5276 t = LLVMInt32Type ();
5278 t = LLVMInt64Type ();
5280 g_assert (ins->inst_offset == 0);
5282 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5283 args [1] = convert (ctx, rhs, t);
5285 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
5288 case OP_ATOMIC_ADD_I4:
5289 case OP_ATOMIC_ADD_I8: {
5290 LLVMValueRef args [2];
5293 if (ins->opcode == OP_ATOMIC_ADD_I4)
5294 t = LLVMInt32Type ();
5296 t = LLVMInt64Type ();
5298 g_assert (ins->inst_offset == 0);
5300 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5301 args [1] = convert (ctx, rhs, t);
5302 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
5305 case OP_ATOMIC_CAS_I4:
5306 case OP_ATOMIC_CAS_I8: {
5307 LLVMValueRef args [3], val;
5310 if (ins->opcode == OP_ATOMIC_CAS_I4)
5311 t = LLVMInt32Type ();
5313 t = LLVMInt64Type ();
5315 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5317 args [1] = convert (ctx, values [ins->sreg3], t);
5319 args [2] = convert (ctx, values [ins->sreg2], t);
5320 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
5321 /* cmpxchg returns a pair */
5322 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
5325 case OP_MEMORY_BARRIER: {
5326 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
5329 case OP_ATOMIC_LOAD_I1:
5330 case OP_ATOMIC_LOAD_I2:
5331 case OP_ATOMIC_LOAD_I4:
5332 case OP_ATOMIC_LOAD_I8:
5333 case OP_ATOMIC_LOAD_U1:
5334 case OP_ATOMIC_LOAD_U2:
5335 case OP_ATOMIC_LOAD_U4:
5336 case OP_ATOMIC_LOAD_U8:
5337 case OP_ATOMIC_LOAD_R4:
5338 case OP_ATOMIC_LOAD_R8: {
5339 set_failure (ctx, "atomic mono.load intrinsic");
5343 gboolean sext, zext;
5345 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5346 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5347 LLVMValueRef index, addr;
5349 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5354 if (ins->inst_offset != 0) {
5355 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5356 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
5361 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5363 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, dname, is_volatile, barrier);
5366 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5368 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5372 case OP_ATOMIC_STORE_I1:
5373 case OP_ATOMIC_STORE_I2:
5374 case OP_ATOMIC_STORE_I4:
5375 case OP_ATOMIC_STORE_I8:
5376 case OP_ATOMIC_STORE_U1:
5377 case OP_ATOMIC_STORE_U2:
5378 case OP_ATOMIC_STORE_U4:
5379 case OP_ATOMIC_STORE_U8:
5380 case OP_ATOMIC_STORE_R4:
5381 case OP_ATOMIC_STORE_R8: {
5382 set_failure (ctx, "atomic mono.store intrinsic");
5386 gboolean sext, zext;
5388 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5389 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5390 LLVMValueRef index, addr, value;
5392 if (!values [ins->inst_destbasereg]) {
5393 set_failure (ctx, "inst_destbasereg");
5397 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5399 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5400 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
5401 value = convert (ctx, values [ins->sreg1], t);
5403 emit_store_general (ctx, bb, &builder, size, value, addr, is_volatile, barrier);
5407 case OP_RELAXED_NOP: {
5408 #if defined(TARGET_AMD64) || defined(TARGET_X86)
5409 emit_call (ctx, bb, &builder, LLVMGetNamedFunction (ctx->lmodule, "llvm.x86.sse2.pause"), NULL, 0);
5416 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
5418 // 257 == FS segment register
5419 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
5421 // 256 == GS segment register
5422 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5425 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
5426 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
5427 /* See mono_amd64_emit_tls_get () */
5428 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
5430 // 256 == GS segment register
5431 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5432 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
5434 set_failure (ctx, "opcode tls-get");
5440 case OP_TLS_GET_REG: {
5441 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5442 /* See emit_tls_get_reg () */
5443 // 256 == GS segment register
5444 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5445 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt32Type ()), ptrtype, ""), "");
5447 set_failure (ctx, "opcode tls-get");
5453 case OP_TLS_SET_REG: {
5454 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5455 /* See emit_tls_get_reg () */
5456 // 256 == GS segment register
5457 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5458 LLVMBuildStore (builder, convert (ctx, lhs, IntPtrType ()), LLVMBuildIntToPtr (builder, convert (ctx, rhs, LLVMInt32Type ()), ptrtype, ""));
5460 set_failure (ctx, "opcode tls-set-reg");
5470 case OP_IADD_OVF_UN:
5472 case OP_ISUB_OVF_UN:
5474 case OP_IMUL_OVF_UN:
5475 #if SIZEOF_VOID_P == 8
5477 case OP_LADD_OVF_UN:
5479 case OP_LSUB_OVF_UN:
5481 case OP_LMUL_OVF_UN:
5484 LLVMValueRef args [2], val, ovf, func;
5486 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
5487 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
5488 func = LLVMGetNamedFunction (lmodule, ovf_op_to_intrins (ins->opcode));
5490 val = LLVMBuildCall (builder, func, args, 2, "");
5491 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
5492 ovf = LLVMBuildExtractValue (builder, val, 1, "");
5493 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
5496 builder = ctx->builder;
5502 * We currently model them using arrays. Promotion to local vregs is
5503 * disabled for them in mono_handle_global_vregs () in the LLVM case,
5504 * so we always have an entry in cfg->varinfo for them.
5505 * FIXME: Is this needed ?
5508 MonoClass *klass = ins->klass;
5509 LLVMValueRef args [5];
5513 set_failure (ctx, "!klass");
5517 if (!addresses [ins->dreg])
5518 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5519 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5520 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5521 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5523 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5524 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5525 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memset_func_name), args, memset_param_count, "");
5528 case OP_DUMMY_VZERO:
5531 case OP_STOREV_MEMBASE:
5532 case OP_LOADV_MEMBASE:
5534 MonoClass *klass = ins->klass;
5535 LLVMValueRef src = NULL, dst, args [5];
5536 gboolean done = FALSE;
5540 set_failure (ctx, "!klass");
5544 if (mini_is_gsharedvt_klass (klass)) {
5546 set_failure (ctx, "gsharedvt");
5550 switch (ins->opcode) {
5551 case OP_STOREV_MEMBASE:
5552 if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg &&
5553 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
5554 /* Decomposed earlier */
5555 g_assert_not_reached ();
5558 if (!addresses [ins->sreg1]) {
5560 g_assert (values [ins->sreg1]);
5561 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
5562 LLVMBuildStore (builder, values [ins->sreg1], dst);
5565 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5566 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5569 case OP_LOADV_MEMBASE:
5570 if (!addresses [ins->dreg])
5571 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5572 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5573 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5576 if (!addresses [ins->sreg1])
5577 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
5578 if (!addresses [ins->dreg])
5579 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5580 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5581 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5584 g_assert_not_reached ();
5594 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5595 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5597 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5598 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5599 LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, memcpy_func_name), args, memcpy_param_count, "");
5602 case OP_LLVM_OUTARG_VT: {
5603 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
5604 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
5606 if (ainfo->storage == LLVMArgScalarByRef) {
5607 LLVMTypeRef argtype;
5608 LLVMValueRef loc, v;
5610 argtype = type_to_llvm_arg_type (ctx, t);
5611 loc = build_alloca_llvm_type (ctx, argtype, 0);
5612 v = convert (ctx, values [ins->sreg1], argtype);
5613 LLVMBuildStore (ctx->builder, v, loc);
5614 addresses [ins->dreg] = loc;
5615 } else if (ainfo->storage == LLVMArgGsharedvtVariable) {
5616 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
5618 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
5619 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
5621 g_assert (addresses [ins->sreg1]);
5622 addresses [ins->dreg] = addresses [ins->sreg1];
5624 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
5625 if (!addresses [ins->sreg1]) {
5626 addresses [ins->sreg1] = build_alloca (ctx, t);
5627 g_assert (values [ins->sreg1]);
5629 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
5630 addresses [ins->dreg] = addresses [ins->sreg1];
5632 if (!addresses [ins->sreg1]) {
5633 addresses [ins->sreg1] = build_alloca (ctx, t);
5634 g_assert (values [ins->sreg1]);
5635 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
5637 addresses [ins->dreg] = addresses [ins->sreg1];
5645 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5647 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
5650 case OP_LOADX_MEMBASE: {
5651 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
5654 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5655 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
5658 case OP_STOREX_MEMBASE: {
5659 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
5662 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5663 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
5670 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
5674 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
5680 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
5684 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
5688 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
5692 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
5695 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
5698 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
5701 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
5705 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
5716 LLVMValueRef v = NULL;
5718 switch (ins->opcode) {
5723 t = LLVMVectorType (LLVMInt32Type (), 4);
5724 rt = LLVMVectorType (LLVMFloatType (), 4);
5730 t = LLVMVectorType (LLVMInt64Type (), 2);
5731 rt = LLVMVectorType (LLVMDoubleType (), 2);
5734 t = LLVMInt32Type ();
5735 rt = LLVMInt32Type ();
5736 g_assert_not_reached ();
5739 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5740 rhs = LLVMBuildBitCast (builder, rhs, t, "");
5741 switch (ins->opcode) {
5744 v = LLVMBuildAnd (builder, lhs, rhs, "");
5748 v = LLVMBuildOr (builder, lhs, rhs, "");
5752 v = LLVMBuildXor (builder, lhs, rhs, "");
5756 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
5759 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
5783 case OP_PADDB_SAT_UN:
5784 case OP_PADDW_SAT_UN:
5785 case OP_PSUBB_SAT_UN:
5786 case OP_PSUBW_SAT_UN:
5794 case OP_PMULW_HIGH_UN: {
5795 LLVMValueRef args [2];
5800 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5807 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, ""), LLVMTypeOf (lhs), "");
5811 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, ""), LLVMTypeOf (lhs), "");
5819 case OP_EXTRACTX_U2:
5821 case OP_EXTRACT_U1: {
5823 gboolean zext = FALSE;
5825 t = simd_op_to_llvm_type (ins->opcode);
5827 switch (ins->opcode) {
5835 case OP_EXTRACTX_U2:
5840 t = LLVMInt32Type ();
5841 g_assert_not_reached ();
5844 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5845 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
5847 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
5856 case OP_EXPAND_R8: {
5857 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
5858 LLVMValueRef mask [16], v;
5861 for (i = 0; i < 16; ++i)
5862 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5864 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
5866 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
5867 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
5872 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5875 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5878 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5881 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5884 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5887 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
5898 case OP_EXTRACT_MASK:
5905 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
5907 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
5913 LLVMValueRef args [3];
5917 args [2] = LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE);
5919 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 3, dname);
5924 /* This is only used for implementing shifts by non-immediate */
5925 values [ins->dreg] = lhs;
5936 LLVMValueRef args [3];
5939 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
5941 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5952 case OP_PSHLQ_REG: {
5953 LLVMValueRef args [3];
5956 args [1] = values [ins->sreg2];
5958 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (lmodule, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5965 case OP_PSHUFLEW_LOW:
5966 case OP_PSHUFLEW_HIGH: {
5968 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
5969 int i, mask_size = 0;
5970 int imask = ins->inst_c0;
5972 /* Convert the x86 shuffle mask to LLVM's */
5973 switch (ins->opcode) {
5976 mask [0] = ((imask >> 0) & 3);
5977 mask [1] = ((imask >> 2) & 3);
5978 mask [2] = ((imask >> 4) & 3) + 4;
5979 mask [3] = ((imask >> 6) & 3) + 4;
5980 v1 = values [ins->sreg1];
5981 v2 = values [ins->sreg2];
5985 mask [0] = ((imask >> 0) & 1);
5986 mask [1] = ((imask >> 1) & 1) + 2;
5987 v1 = values [ins->sreg1];
5988 v2 = values [ins->sreg2];
5990 case OP_PSHUFLEW_LOW:
5992 mask [0] = ((imask >> 0) & 3);
5993 mask [1] = ((imask >> 2) & 3);
5994 mask [2] = ((imask >> 4) & 3);
5995 mask [3] = ((imask >> 6) & 3);
6000 v1 = values [ins->sreg1];
6001 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6003 case OP_PSHUFLEW_HIGH:
6009 mask [4] = 4 + ((imask >> 0) & 3);
6010 mask [5] = 4 + ((imask >> 2) & 3);
6011 mask [6] = 4 + ((imask >> 4) & 3);
6012 mask [7] = 4 + ((imask >> 6) & 3);
6013 v1 = values [ins->sreg1];
6014 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6018 mask [0] = ((imask >> 0) & 3);
6019 mask [1] = ((imask >> 2) & 3);
6020 mask [2] = ((imask >> 4) & 3);
6021 mask [3] = ((imask >> 6) & 3);
6022 v1 = values [ins->sreg1];
6023 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6026 g_assert_not_reached ();
6028 for (i = 0; i < mask_size; ++i)
6029 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6031 values [ins->dreg] =
6032 LLVMBuildShuffleVector (builder, v1, v2,
6033 LLVMConstVector (mask_values, mask_size), dname);
6037 case OP_UNPACK_LOWB:
6038 case OP_UNPACK_LOWW:
6039 case OP_UNPACK_LOWD:
6040 case OP_UNPACK_LOWQ:
6041 case OP_UNPACK_LOWPS:
6042 case OP_UNPACK_LOWPD:
6043 case OP_UNPACK_HIGHB:
6044 case OP_UNPACK_HIGHW:
6045 case OP_UNPACK_HIGHD:
6046 case OP_UNPACK_HIGHQ:
6047 case OP_UNPACK_HIGHPS:
6048 case OP_UNPACK_HIGHPD: {
6050 LLVMValueRef mask_values [16];
6051 int i, mask_size = 0;
6052 gboolean low = FALSE;
6054 switch (ins->opcode) {
6055 case OP_UNPACK_LOWB:
6059 case OP_UNPACK_LOWW:
6063 case OP_UNPACK_LOWD:
6064 case OP_UNPACK_LOWPS:
6068 case OP_UNPACK_LOWQ:
6069 case OP_UNPACK_LOWPD:
6073 case OP_UNPACK_HIGHB:
6076 case OP_UNPACK_HIGHW:
6079 case OP_UNPACK_HIGHD:
6080 case OP_UNPACK_HIGHPS:
6083 case OP_UNPACK_HIGHQ:
6084 case OP_UNPACK_HIGHPD:
6088 g_assert_not_reached ();
6092 for (i = 0; i < (mask_size / 2); ++i) {
6094 mask [(i * 2) + 1] = mask_size + i;
6097 for (i = 0; i < (mask_size / 2); ++i) {
6098 mask [(i * 2)] = (mask_size / 2) + i;
6099 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
6103 for (i = 0; i < mask_size; ++i)
6104 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6106 values [ins->dreg] =
6107 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
6108 LLVMConstVector (mask_values, mask_size), dname);
6113 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6114 LLVMValueRef v, val;
6116 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6117 val = LLVMConstNull (t);
6118 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6119 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
6121 values [ins->dreg] = val;
6125 case OP_DUPPS_HIGH: {
6126 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6127 LLVMValueRef v1, v2, val;
6130 if (ins->opcode == OP_DUPPS_LOW) {
6131 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6132 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6134 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6135 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6137 val = LLVMConstNull (t);
6138 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6139 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6140 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6141 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6143 values [ins->dreg] = val;
6153 * EXCEPTION HANDLING
6155 case OP_IMPLICIT_EXCEPTION:
6156 /* This marks a place where an implicit exception can happen */
6157 if (bb->region != -1)
6158 set_failure (ctx, "implicit-exception");
6162 gboolean rethrow = (ins->opcode == OP_RETHROW);
6163 if (ctx->llvm_only) {
6164 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
6165 has_terminator = TRUE;
6166 ctx->unreachable [bb->block_num] = TRUE;
6168 emit_throw (ctx, bb, rethrow, lhs);
6169 builder = ctx->builder;
6173 case OP_CALL_HANDLER: {
6175 * We don't 'call' handlers, but instead simply branch to them.
6176 * The code generated by ENDFINALLY will branch back to us.
6178 LLVMBasicBlockRef noex_bb;
6180 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
6182 bb_list = info->call_handler_return_bbs;
6185 * Set the indicator variable for the finally clause.
6187 lhs = info->finally_ind;
6189 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
6191 /* Branch to the finally clause */
6192 LLVMBuildBr (builder, info->call_handler_target_bb);
6194 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
6195 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
6197 builder = ctx->builder = create_builder (ctx);
6198 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
6200 bblocks [bb->block_num].end_bblock = noex_bb;
6203 case OP_START_HANDLER: {
6206 case OP_ENDFINALLY: {
6207 LLVMBasicBlockRef resume_bb;
6208 MonoBasicBlock *handler_bb;
6209 LLVMValueRef val, switch_ins, callee;
6213 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
6214 g_assert (handler_bb);
6215 info = &bblocks [handler_bb->block_num];
6216 lhs = info->finally_ind;
6219 bb_list = info->call_handler_return_bbs;
6221 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
6223 /* Load the finally variable */
6224 val = LLVMBuildLoad (builder, lhs, "");
6226 /* Reset the variable */
6227 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
6229 /* Branch to either resume_bb, or to the bblocks in bb_list */
6230 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
6232 * The other targets are added at the end to handle OP_CALL_HANDLER
6233 * opcodes processed later.
6235 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
6237 builder = ctx->builder = create_builder (ctx);
6238 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
6240 if (ctx->llvm_only) {
6241 emit_resume_eh (ctx, bb);
6243 if (ctx->cfg->compile_aot) {
6244 callee = get_callee (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline");
6246 callee = LLVMGetNamedFunction (lmodule, "llvm_resume_unwind_trampoline");
6248 LLVMBuildCall (builder, callee, NULL, 0, "");
6249 LLVMBuildUnreachable (builder);
6252 has_terminator = TRUE;
6255 case OP_IL_SEQ_POINT:
6260 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
6261 set_failure (ctx, reason);
6269 /* Convert the value to the type required by phi nodes */
6270 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
6271 if (!values [ins->dreg])
6273 values [ins->dreg] = addresses [ins->dreg];
6275 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
6278 /* Add stores for volatile variables */
6279 if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
6280 emit_volatile_store (ctx, ins->dreg);
6286 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
6287 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
6290 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
6291 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
6292 LLVMBuildRetVoid (builder);
6295 if (bb == cfg->bb_entry)
6296 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
6300 * mono_llvm_check_method_supported:
6302 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
6303 * compiling a method twice.
6306 mono_llvm_check_method_supported (MonoCompile *cfg)
6313 if (cfg->method->save_lmf) {
6314 cfg->exception_message = g_strdup ("lmf");
6315 cfg->disable_llvm = TRUE;
6317 if (cfg->disable_llvm)
6321 * Nested clauses where one of the clauses is a finally clause is
6322 * not supported, because LLVM can't figure out the control flow,
6323 * probably because we resume exception handling by calling our
6324 * own function instead of using the 'resume' llvm instruction.
6326 for (i = 0; i < cfg->header->num_clauses; ++i) {
6327 for (j = 0; j < cfg->header->num_clauses; ++j) {
6328 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
6329 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
6331 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
6332 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
6333 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
6334 cfg->exception_message = g_strdup ("nested clauses");
6335 cfg->disable_llvm = TRUE;
6340 if (cfg->disable_llvm)
6344 if (cfg->method->dynamic) {
6345 cfg->exception_message = g_strdup ("dynamic.");
6346 cfg->disable_llvm = TRUE;
6348 if (cfg->disable_llvm)
6352 static LLVMCallInfo*
6353 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
6355 LLVMCallInfo *linfo;
6358 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
6362 * Gsharedvt methods have the following calling convention:
6363 * - all arguments are passed by ref, even non generic ones
6364 * - the return value is returned by ref too, using a vret
6365 * argument passed after 'this'.
6367 n = sig->param_count + sig->hasthis;
6368 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
6372 linfo->args [pindex ++].storage = LLVMArgNormal;
6374 if (sig->ret->type != MONO_TYPE_VOID) {
6375 if (mini_is_gsharedvt_variable_type (sig->ret))
6376 linfo->ret.storage = LLVMArgGsharedvtVariable;
6377 else if (mini_type_is_vtype (sig->ret))
6378 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
6380 linfo->ret.storage = LLVMArgGsharedvtFixed;
6381 linfo->vret_arg_index = pindex;
6383 linfo->ret.storage = LLVMArgNone;
6386 for (i = 0; i < sig->param_count; ++i) {
6387 if (sig->params [i]->byref)
6388 linfo->args [pindex].storage = LLVMArgNormal;
6389 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
6390 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
6391 else if (mini_type_is_vtype (sig->params [i]))
6392 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
6394 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
6395 linfo->args [pindex].type = sig->params [i];
6402 linfo = mono_arch_get_llvm_call_info (cfg, sig);
6403 for (i = 0; i < sig->param_count; ++i)
6404 linfo->args [i + sig->hasthis].type = sig->params [i];
6410 emit_method_inner (EmitContext *ctx);
6413 free_ctx (EmitContext *ctx)
6417 g_free (ctx->values);
6418 g_free (ctx->addresses);
6419 g_free (ctx->vreg_types);
6420 g_free (ctx->vreg_cli_types);
6421 g_free (ctx->is_dead);
6422 g_free (ctx->unreachable);
6423 g_ptr_array_free (ctx->phi_values, TRUE);
6424 g_free (ctx->bblocks);
6425 g_hash_table_destroy (ctx->region_to_handler);
6426 g_hash_table_destroy (ctx->clause_to_handler);
6427 g_free (ctx->method_name);
6428 g_ptr_array_free (ctx->bblock_list, TRUE);
6430 for (l = ctx->builders; l; l = l->next) {
6431 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
6432 LLVMDisposeBuilder (builder);
6439 * mono_llvm_emit_method:
6441 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
6444 mono_llvm_emit_method (MonoCompile *cfg)
6448 gboolean is_linkonce = FALSE;
6451 /* The code below might acquire the loader lock, so use it for global locking */
6452 mono_loader_lock ();
6454 /* Used to communicate with the callbacks */
6455 mono_native_tls_set_value (current_cfg_tls_id, cfg);
6457 ctx = g_new0 (EmitContext, 1);
6459 ctx->mempool = cfg->mempool;
6462 * This maps vregs to the LLVM instruction defining them
6464 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
6466 * This maps vregs for volatile variables to the LLVM instruction defining their
6469 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
6470 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
6471 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
6472 ctx->phi_values = g_ptr_array_sized_new (256);
6474 * This signals whenever the vreg was defined by a phi node with no input vars
6475 * (i.e. all its input bblocks end with NOT_REACHABLE).
6477 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
6478 /* Whenever the bblock is unreachable */
6479 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
6480 ctx->bblock_list = g_ptr_array_sized_new (256);
6482 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
6483 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
6484 ctx->method_to_callers = g_hash_table_new (NULL, NULL);
6485 if (cfg->compile_aot) {
6486 ctx->module = &aot_module;
6490 * Allow the linker to discard duplicate copies of wrappers, generic instances etc. by using the 'linkonce'
6491 * linkage for them. This requires the following:
6492 * - the method needs to have a unique mangled name
6493 * - llvmonly mode, since the code in aot-runtime.c would initialize got slots in the wrong aot image etc.
6495 is_linkonce = ctx->module->llvm_only && ctx->module->static_link && mono_aot_is_linkonce_method (cfg->method);
6497 method_name = mono_aot_get_mangled_method_name (cfg->method);
6499 is_linkonce = FALSE;
6502 printf ("%s %s\n", mono_method_full_name (cfg->method, 1), method_name);
6504 printf ("%s\n", mono_method_full_name (cfg->method, 1));
6508 method_name = mono_aot_get_method_name (cfg);
6509 cfg->llvm_method_name = g_strdup (method_name);
6511 init_jit_module (cfg->domain);
6512 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
6513 method_name = mono_method_full_name (cfg->method, TRUE);
6515 ctx->method_name = method_name;
6516 ctx->is_linkonce = is_linkonce;
6518 ctx->lmodule = ctx->module->lmodule;
6519 ctx->llvm_only = ctx->module->llvm_only;
6521 emit_method_inner (ctx);
6523 if (!ctx_ok (ctx)) {
6525 /* Need to add unused phi nodes as they can be referenced by other values */
6526 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
6527 LLVMBuilderRef builder;
6529 builder = create_builder (ctx);
6530 LLVMPositionBuilderAtEnd (builder, phi_bb);
6532 for (i = 0; i < ctx->phi_values->len; ++i) {
6533 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
6534 if (LLVMGetInstructionParent (v) == NULL)
6535 LLVMInsertIntoBuilder (builder, v);
6538 LLVMDeleteFunction (ctx->lmethod);
6544 mono_native_tls_set_value (current_cfg_tls_id, NULL);
6546 mono_loader_unlock ();
6550 emit_method_inner (EmitContext *ctx)
6552 MonoCompile *cfg = ctx->cfg;
6553 MonoMethodSignature *sig;
6555 LLVMTypeRef method_type;
6556 LLVMValueRef method = NULL;
6557 LLVMValueRef *values = ctx->values;
6558 int i, max_block_num, bb_index;
6559 gboolean last = FALSE;
6560 LLVMCallInfo *linfo;
6561 LLVMModuleRef lmodule = ctx->lmodule;
6563 GPtrArray *bblock_list = ctx->bblock_list;
6564 MonoMethodHeader *header;
6565 MonoExceptionClause *clause;
6568 if (cfg->gsharedvt && !cfg->llvm_only) {
6569 set_failure (ctx, "gsharedvt");
6575 static int count = 0;
6578 if (g_getenv ("LLVM_COUNT")) {
6579 if (count == atoi (g_getenv ("LLVM_COUNT"))) {
6580 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
6584 if (count > atoi (g_getenv ("LLVM_COUNT"))) {
6585 set_failure (ctx, "count");
6592 sig = mono_method_signature (cfg->method);
6595 linfo = get_llvm_call_info (cfg, sig);
6601 linfo->rgctx_arg = TRUE;
6602 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
6606 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
6607 ctx->lmethod = method;
6609 if (!cfg->llvm_only)
6610 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
6611 LLVMSetLinkage (method, LLVMPrivateLinkage);
6613 LLVMAddFunctionAttr (method, LLVMUWTable);
6615 if (cfg->compile_aot) {
6616 LLVMSetLinkage (method, LLVMInternalLinkage);
6617 if (ctx->module->external_symbols) {
6618 LLVMSetLinkage (method, LLVMExternalLinkage);
6619 LLVMSetVisibility (method, LLVMHiddenVisibility);
6621 if (ctx->is_linkonce) {
6622 LLVMSetLinkage (method, LLVMLinkOnceAnyLinkage);
6623 LLVMSetVisibility (method, LLVMDefaultVisibility);
6626 LLVMSetLinkage (method, LLVMPrivateLinkage);
6629 if (cfg->method->save_lmf && !cfg->llvm_only) {
6630 set_failure (ctx, "lmf");
6634 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
6635 set_failure (ctx, "pinvoke signature");
6639 header = cfg->header;
6640 for (i = 0; i < header->num_clauses; ++i) {
6641 clause = &header->clauses [i];
6642 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
6643 set_failure (ctx, "non-finally/catch clause.");
6647 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
6648 /* We can't handle inlined methods with clauses */
6649 LLVMAddFunctionAttr (method, LLVMNoInlineAttribute);
6651 if (linfo->rgctx_arg) {
6652 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
6653 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
6655 * We mark the rgctx parameter with the inreg attribute, which is mapped to
6656 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
6657 * CC_X86_64_Mono in X86CallingConv.td.
6659 if (!ctx->llvm_only)
6660 LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute);
6661 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
6663 ctx->rgctx_arg_pindex = -1;
6665 if (cfg->vret_addr) {
6666 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
6667 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
6668 if (linfo->ret.storage == LLVMArgVtypeByRef) {
6669 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMStructRetAttribute);
6670 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
6672 } else if (linfo->ret.storage == LLVMArgScalarRetAddr) {
6673 LLVMValueRef param = LLVMGetParam (method, linfo->vret_arg_pindex);
6674 LLVMSetValueName (param, "vret");
6678 ctx->this_arg_pindex = linfo->this_arg_pindex;
6679 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
6680 values [cfg->args [0]->dreg] = ctx->this_arg;
6681 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
6684 names = g_new (char *, sig->param_count);
6685 mono_method_get_param_names (cfg->method, (const char **) names);
6687 for (i = 0; i < sig->param_count; ++i) {
6688 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
6690 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
6693 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
6694 name = g_strdup_printf ("dummy_%d_%d", i, j);
6695 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
6699 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
6700 if (ainfo->storage == LLVMArgScalarByRef) {
6701 if (names [i] && names [i][0] != '\0')
6702 name = g_strdup_printf ("p_arg_%s", names [i]);
6704 name = g_strdup_printf ("p_arg_%d", i);
6705 } else if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
6706 if (names [i] && names [i][0] != '\0')
6707 name = g_strdup_printf ("p_arg_%s", names [i]);
6709 name = g_strdup_printf ("p_arg_%d", i);
6711 if (names [i] && names [i][0] != '\0')
6712 name = g_strdup_printf ("arg_%s", names [i]);
6714 name = g_strdup_printf ("arg_%d", i);
6716 LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name);
6718 if (ainfo->storage == LLVMArgVtypeByVal)
6719 LLVMAddAttribute (LLVMGetParam (method, pindex), LLVMByValAttribute);
6721 if (ainfo->storage == LLVMArgVtypeByRef) {
6723 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
6728 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
6729 ctx->minfo = mono_debug_lookup_method (cfg->method);
6730 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
6734 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
6735 max_block_num = MAX (max_block_num, bb->block_num);
6736 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
6738 /* Add branches between non-consecutive bblocks */
6739 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6740 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
6741 bb->next_bb != bb->last_ins->inst_false_bb) {
6743 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
6744 inst->opcode = OP_BR;
6745 inst->inst_target_bb = bb->last_ins->inst_false_bb;
6746 mono_bblock_add_inst (bb, inst);
6751 * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR
6752 * was later optimized away, so clear these flags, and add them back for the still
6753 * present OP_LDADDR instructions.
6755 for (i = 0; i < cfg->next_vreg; ++i) {
6758 ins = get_vreg_to_inst (cfg, i);
6759 if (ins && ins != cfg->rgctx_var)
6760 ins->flags &= ~MONO_INST_INDIRECT;
6764 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
6766 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6768 LLVMBuilderRef builder;
6770 char dname_buf[128];
6772 builder = create_builder (ctx);
6774 for (ins = bb->code; ins; ins = ins->next) {
6775 switch (ins->opcode) {
6780 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, &ins->klass->byval_arg));
6785 if (ins->opcode == OP_VPHI) {
6786 /* Treat valuetype PHI nodes as operating on the address itself */
6787 g_assert (ins->klass);
6788 phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
6792 * Have to precreate these, as they can be referenced by
6793 * earlier instructions.
6795 sprintf (dname_buf, "t%d", ins->dreg);
6797 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
6799 if (ins->opcode == OP_VPHI)
6800 ctx->addresses [ins->dreg] = values [ins->dreg];
6802 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
6805 * Set the expected type of the incoming arguments since these have
6806 * to have the same type.
6808 for (i = 0; i < ins->inst_phi_args [0]; i++) {
6809 int sreg1 = ins->inst_phi_args [i + 1];
6812 ctx->vreg_types [sreg1] = phi_type;
6817 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
6826 * Create an ordering for bblocks, use the depth first order first, then
6827 * put the exception handling bblocks last.
6829 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
6830 bb = cfg->bblocks [bb_index];
6831 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
6832 g_ptr_array_add (bblock_list, bb);
6833 bblocks [bb->block_num].added = TRUE;
6837 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6838 if (!bblocks [bb->block_num].added)
6839 g_ptr_array_add (bblock_list, bb);
6843 * Second pass: generate code.
6846 LLVMBuilderRef entry_builder = create_builder (ctx);
6847 LLVMBasicBlockRef entry_bb = get_bb (ctx, cfg->bb_entry);
6848 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
6849 emit_entry_bb (ctx, entry_builder);
6851 // Make landing pads first
6852 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
6854 if (ctx->llvm_only) {
6855 size_t group_index = 0;
6856 while (group_index < cfg->header->num_clauses) {
6858 size_t cursor = group_index;
6859 while (cursor < cfg->header->num_clauses &&
6860 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
6861 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
6866 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
6867 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
6868 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
6870 group_index = cursor;
6874 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
6875 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
6877 // Prune unreachable mono BBs.
6878 if (!(bb == cfg->bb_entry || bb->in_count > 0))
6881 process_bb (ctx, bb);
6885 g_hash_table_destroy (ctx->exc_meta);
6887 mono_memory_barrier ();
6889 /* Add incoming phi values */
6890 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6891 GSList *l, *ins_list;
6893 ins_list = bblocks [bb->block_num].phi_nodes;
6895 for (l = ins_list; l; l = l->next) {
6896 PhiNode *node = (PhiNode*)l->data;
6897 MonoInst *phi = node->phi;
6898 int sreg1 = node->sreg;
6899 LLVMBasicBlockRef in_bb;
6904 in_bb = get_end_bb (ctx, node->in_bb);
6906 if (ctx->unreachable [node->in_bb->block_num])
6909 if (!values [sreg1]) {
6910 /* Can happen with values in EH clauses */
6911 set_failure (ctx, "incoming phi sreg1");
6915 if (phi->opcode == OP_VPHI) {
6916 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
6917 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
6919 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
6920 set_failure (ctx, "incoming phi arg type mismatch");
6923 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
6924 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
6929 /* Nullify empty phi instructions */
6930 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6931 GSList *l, *ins_list;
6933 ins_list = bblocks [bb->block_num].phi_nodes;
6935 for (l = ins_list; l; l = l->next) {
6936 PhiNode *node = (PhiNode*)l->data;
6937 MonoInst *phi = node->phi;
6938 LLVMValueRef phi_ins = values [phi->dreg];
6941 /* Already removed */
6944 if (LLVMCountIncoming (phi_ins) == 0) {
6945 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
6946 LLVMInstructionEraseFromParent (phi_ins);
6947 values [phi->dreg] = NULL;
6952 /* Create the SWITCH statements for ENDFINALLY instructions */
6953 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
6954 BBInfo *info = &bblocks [bb->block_num];
6956 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
6957 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
6958 GSList *bb_list = info->call_handler_return_bbs;
6960 for (i = 0; i < g_slist_length (bb_list); ++i)
6961 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)(g_slist_nth (bb_list, i)->data));
6965 /* Initialize the method if needed */
6966 if (cfg->compile_aot && ctx->llvm_only) {
6967 // FIXME: Add more shared got entries
6968 ctx->builder = create_builder (ctx);
6969 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
6971 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
6973 // FIXME: beforefieldinit
6974 if (ctx->has_got_access || mono_class_get_cctor (cfg->method->klass)) {
6975 emit_init_method (ctx);
6977 LLVMBuildBr (ctx->builder, ctx->inited_bb);
6981 if (cfg->llvm_only) {
6982 GHashTableIter iter;
6984 GSList *callers, *l, *l2;
6987 * Add the contents of ctx->method_to_callers to module->method_to_callers.
6988 * We can't do this earlier, as it contains llvm instructions which can be
6989 * freed if compilation fails.
6990 * FIXME: Get rid of this when all methods can be llvm compiled.
6992 g_hash_table_iter_init (&iter, ctx->method_to_callers);
6993 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
6994 for (l = callers; l; l = l->next) {
6995 l2 = (GSList*)g_hash_table_lookup (ctx->module->method_to_callers, method);
6996 l2 = g_slist_prepend (l2, l->data);
6997 g_hash_table_insert (ctx->module->method_to_callers, method, l2);
7002 if (cfg->verbose_level > 1)
7003 mono_llvm_dump_value (method);
7005 if (cfg->compile_aot && !cfg->llvm_only)
7006 mark_as_used (ctx->module, method);
7008 if (cfg->compile_aot && !cfg->llvm_only) {
7009 LLVMValueRef md_args [16];
7010 LLVMValueRef md_node;
7013 method_index = mono_aot_get_method_index (cfg->orig_method);
7014 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
7015 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
7016 md_node = LLVMMDNode (md_args, 2);
7017 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
7018 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
7021 if (cfg->compile_aot) {
7022 /* Don't generate native code, keep the LLVM IR */
7023 if (cfg->verbose_level)
7024 printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), ctx->method_name);
7026 int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
7027 g_assert (err == 0);
7029 //LLVMVerifyFunction(method, 0);
7030 mono_llvm_optimize_method (ctx->module->mono_ee, ctx->lmethod);
7032 if (cfg->verbose_level > 1)
7033 mono_llvm_dump_value (ctx->lmethod);
7035 cfg->native_code = (unsigned char*)LLVMGetPointerToGlobal (ctx->module->ee, ctx->lmethod);
7037 /* Set by emit_cb */
7038 g_assert (cfg->code_len);
7041 if (ctx->module->method_to_lmethod)
7042 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
7043 if (ctx->module->idx_to_lmethod)
7044 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
7046 if (ctx->llvm_only && cfg->orig_method->klass->valuetype && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
7047 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
7051 * mono_llvm_create_vars:
7053 * Same as mono_arch_create_vars () for LLVM.
7056 mono_llvm_create_vars (MonoCompile *cfg)
7058 MonoMethodSignature *sig;
7060 sig = mono_method_signature (cfg->method);
7061 if (cfg->gsharedvt && cfg->llvm_only) {
7062 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
7063 cfg->vret_addr = mono_compile_create_var (cfg, &mono_get_intptr_class ()->byval_arg, OP_ARG);
7064 if (G_UNLIKELY (cfg->verbose_level > 1)) {
7065 printf ("vret_addr = ");
7066 mono_print_ins (cfg->vret_addr);
7070 mono_arch_create_vars (cfg);
7075 * mono_llvm_emit_call:
7077 * Same as mono_arch_emit_call () for LLVM.
7080 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
7083 MonoMethodSignature *sig;
7084 int i, n, stack_size;
7089 sig = call->signature;
7090 n = sig->param_count + sig->hasthis;
7092 call->cinfo = get_llvm_call_info (cfg, sig);
7094 if (cfg->disable_llvm)
7097 if (sig->call_convention == MONO_CALL_VARARG) {
7098 cfg->exception_message = g_strdup ("varargs");
7099 cfg->disable_llvm = TRUE;
7102 for (i = 0; i < n; ++i) {
7105 ainfo = call->cinfo->args + i;
7107 in = call->args [i];
7109 /* Simply remember the arguments */
7110 switch (ainfo->storage) {
7111 case LLVMArgNormal: {
7112 MonoType *t = (sig->hasthis && i == 0) ? &mono_get_intptr_class ()->byval_arg : ainfo->type;
7115 opcode = mono_type_to_regmove (cfg, t);
7116 if (opcode == OP_FMOVE) {
7117 MONO_INST_NEW (cfg, ins, OP_FMOVE);
7118 ins->dreg = mono_alloc_freg (cfg);
7119 } else if (opcode == OP_LMOVE) {
7120 MONO_INST_NEW (cfg, ins, OP_LMOVE);
7121 ins->dreg = mono_alloc_lreg (cfg);
7122 } else if (opcode == OP_RMOVE) {
7123 MONO_INST_NEW (cfg, ins, OP_RMOVE);
7124 ins->dreg = mono_alloc_freg (cfg);
7126 MONO_INST_NEW (cfg, ins, OP_MOVE);
7127 ins->dreg = mono_alloc_ireg (cfg);
7129 ins->sreg1 = in->dreg;
7132 case LLVMArgVtypeByVal:
7133 case LLVMArgVtypeByRef:
7134 case LLVMArgVtypeInReg:
7135 case LLVMArgVtypeAsScalar:
7136 case LLVMArgScalarByRef:
7137 case LLVMArgAsIArgs:
7138 case LLVMArgAsFpArgs:
7139 case LLVMArgGsharedvtVariable:
7140 case LLVMArgGsharedvtFixed:
7141 case LLVMArgGsharedvtFixedVtype:
7142 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
7143 ins->dreg = mono_alloc_ireg (cfg);
7144 ins->sreg1 = in->dreg;
7145 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
7146 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
7147 ins->inst_vtype = ainfo->type;
7148 ins->klass = mono_class_from_mono_type (ainfo->type);
7151 cfg->exception_message = g_strdup ("ainfo->storage");
7152 cfg->disable_llvm = TRUE;
7156 if (!cfg->disable_llvm) {
7157 MONO_ADD_INS (cfg->cbb, ins);
7158 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
7163 static unsigned char*
7164 alloc_cb (LLVMValueRef function, int size)
7168 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7172 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
7174 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
7179 emitted_cb (LLVMValueRef function, void *start, void *end)
7183 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7185 cfg->code_len = (guint8*)end - (guint8*)start;
7189 exception_cb (void *data)
7192 MonoJitExceptionInfo *ei;
7193 guint32 ei_len, i, j, nested_len, nindex;
7194 gpointer *type_info;
7195 int this_reg, this_offset;
7197 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7201 * data points to a DWARF FDE structure, convert it to our unwind format and
7203 * An alternative would be to save it directly, and modify our unwinder to work
7206 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
7207 if (cfg->verbose_level > 1)
7208 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
7210 /* Count nested clauses */
7212 for (i = 0; i < ei_len; ++i) {
7213 gint32 cindex1 = *(gint32*)type_info [i];
7214 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
7216 for (j = 0; j < cfg->header->num_clauses; ++j) {
7218 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
7220 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7226 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
7227 cfg->llvm_ex_info_len = ei_len + nested_len;
7228 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
7229 /* Fill the rest of the information from the type info */
7230 for (i = 0; i < ei_len; ++i) {
7231 gint32 clause_index = *(gint32*)type_info [i];
7232 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
7234 cfg->llvm_ex_info [i].flags = clause->flags;
7235 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
7236 cfg->llvm_ex_info [i].clause_index = clause_index;
7240 * For nested clauses, the LLVM produced exception info associates the try interval with
7241 * the innermost handler, while mono expects it to be associated with all nesting clauses.
7242 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
7243 * and everything else from the nested clause.
7246 for (i = 0; i < ei_len; ++i) {
7247 gint32 cindex1 = *(gint32*)type_info [i];
7248 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
7250 for (j = 0; j < cfg->header->num_clauses; ++j) {
7252 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
7253 MonoJitExceptionInfo *nesting_ei, *nested_ei;
7255 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7256 /* clause1 is the nested clause */
7257 nested_ei = &cfg->llvm_ex_info [i];
7258 nesting_ei = &cfg->llvm_ex_info [nindex];
7261 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
7263 nesting_ei->flags = clause2->flags;
7264 nesting_ei->data.catch_class = clause2->data.catch_class;
7265 nesting_ei->clause_index = cindex2;
7269 g_assert (nindex == ei_len + nested_len);
7270 cfg->llvm_this_reg = this_reg;
7271 cfg->llvm_this_offset = this_offset;
7273 /* type_info [i] is cfg mempool allocated, no need to free it */
7280 dlsym_cb (const char *name, void **symbol)
7286 if (!strcmp (name, "__bzero")) {
7287 *symbol = (void*)bzero;
7289 current = mono_dl_open (NULL, 0, NULL);
7292 err = mono_dl_symbol (current, name, symbol);
7294 mono_dl_close (current);
7296 #ifdef MONO_ARCH_HAVE_CREATE_LLVM_NATIVE_THUNK
7297 *symbol = (char*)mono_arch_create_llvm_native_thunk (mono_domain_get (), (guint8*)(*symbol));
7303 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
7305 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
7309 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
7311 LLVMTypeRef param_types [4];
7313 param_types [0] = param_type1;
7314 param_types [1] = param_type2;
7316 AddFunc (module, name, ret_type, param_types, 2);
7320 add_intrinsics (LLVMModuleRef module)
7322 /* Emit declarations of instrinsics */
7324 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
7325 * type doesn't seem to do any locking.
7328 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
7330 memset_param_count = 5;
7331 memset_func_name = "llvm.memset.p0i8.i32";
7333 AddFunc (module, memset_func_name, LLVMVoidType (), params, memset_param_count);
7337 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
7339 memcpy_param_count = 5;
7340 memcpy_func_name = "llvm.memcpy.p0i8.p0i8.i32";
7342 AddFunc (module, memcpy_func_name, LLVMVoidType (), params, memcpy_param_count);
7346 LLVMTypeRef params [] = { LLVMDoubleType () };
7348 AddFunc (module, "llvm.sin.f64", LLVMDoubleType (), params, 1);
7349 AddFunc (module, "llvm.cos.f64", LLVMDoubleType (), params, 1);
7350 AddFunc (module, "llvm.sqrt.f64", LLVMDoubleType (), params, 1);
7352 /* This isn't an intrinsic, instead llvm seems to special case it by name */
7353 AddFunc (module, "fabs", LLVMDoubleType (), params, 1);
7357 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
7358 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
7359 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
7361 AddFunc (module, "llvm.sadd.with.overflow.i32", ret_type, params, 2);
7362 AddFunc (module, "llvm.sadd.with.overflow.i32", ret_type, params, 2);
7363 AddFunc (module, "llvm.uadd.with.overflow.i32", ret_type, params, 2);
7364 AddFunc (module, "llvm.ssub.with.overflow.i32", ret_type, params, 2);
7365 AddFunc (module, "llvm.usub.with.overflow.i32", ret_type, params, 2);
7366 AddFunc (module, "llvm.smul.with.overflow.i32", ret_type, params, 2);
7367 AddFunc (module, "llvm.umul.with.overflow.i32", ret_type, params, 2);
7371 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
7372 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
7373 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
7375 AddFunc (module, "llvm.sadd.with.overflow.i64", ret_type, params, 2);
7376 AddFunc (module, "llvm.uadd.with.overflow.i64", ret_type, params, 2);
7377 AddFunc (module, "llvm.ssub.with.overflow.i64", ret_type, params, 2);
7378 AddFunc (module, "llvm.usub.with.overflow.i64", ret_type, params, 2);
7379 AddFunc (module, "llvm.smul.with.overflow.i64", ret_type, params, 2);
7380 AddFunc (module, "llvm.umul.with.overflow.i64", ret_type, params, 2);
7383 AddFunc2 (module, "llvm.expect.i8", LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
7384 AddFunc2 (module, "llvm.expect.i1", LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
7388 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
7390 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
7393 /* SSE intrinsics */
7394 #if defined(TARGET_X86) || defined(TARGET_AMD64)
7396 LLVMTypeRef ret_type, arg_types [16];
7399 ret_type = type_to_simd_type (MONO_TYPE_I4);
7400 arg_types [0] = ret_type;
7401 arg_types [1] = ret_type;
7402 AddFunc (module, "llvm.x86.sse41.pminud", ret_type, arg_types, 2);
7403 AddFunc (module, "llvm.x86.sse41.pmaxud", ret_type, arg_types, 2);
7405 ret_type = type_to_simd_type (MONO_TYPE_I2);
7406 arg_types [0] = ret_type;
7407 arg_types [1] = ret_type;
7408 AddFunc (module, "llvm.x86.sse41.pminuw", ret_type, arg_types, 2);
7409 AddFunc (module, "llvm.x86.sse2.pmins.w", ret_type, arg_types, 2);
7410 AddFunc (module, "llvm.x86.sse41.pmaxuw", ret_type, arg_types, 2);
7411 AddFunc (module, "llvm.x86.sse2.padds.w", ret_type, arg_types, 2);
7412 AddFunc (module, "llvm.x86.sse2.psubs.w", ret_type, arg_types, 2);
7413 AddFunc (module, "llvm.x86.sse2.paddus.w", ret_type, arg_types, 2);
7414 AddFunc (module, "llvm.x86.sse2.psubus.w", ret_type, arg_types, 2);
7415 AddFunc (module, "llvm.x86.sse2.pavg.w", ret_type, arg_types, 2);
7416 AddFunc (module, "llvm.x86.sse2.pmulh.w", ret_type, arg_types, 2);
7417 AddFunc (module, "llvm.x86.sse2.pmulhu.w", ret_type, arg_types, 2);
7419 ret_type = type_to_simd_type (MONO_TYPE_I1);
7420 arg_types [0] = ret_type;
7421 arg_types [1] = ret_type;
7422 AddFunc (module, "llvm.x86.sse2.pminu.b", ret_type, arg_types, 2);
7423 AddFunc (module, "llvm.x86.sse2.pmaxu.b", ret_type, arg_types, 2);
7424 AddFunc (module, "llvm.x86.sse2.padds.b", ret_type, arg_types, 2);
7425 AddFunc (module, "llvm.x86.sse2.psubs.b", ret_type, arg_types, 2);
7426 AddFunc (module, "llvm.x86.sse2.paddus.b", ret_type, arg_types, 2);
7427 AddFunc (module, "llvm.x86.sse2.psubus.b", ret_type, arg_types, 2);
7428 AddFunc (module, "llvm.x86.sse2.pavg.b", ret_type, arg_types, 2);
7430 ret_type = type_to_simd_type (MONO_TYPE_R8);
7431 arg_types [0] = ret_type;
7432 arg_types [1] = ret_type;
7433 AddFunc (module, "llvm.x86.sse2.min.pd", ret_type, arg_types, 2);
7434 AddFunc (module, "llvm.x86.sse2.max.pd", ret_type, arg_types, 2);
7435 AddFunc (module, "llvm.x86.sse3.hadd.pd", ret_type, arg_types, 2);
7436 AddFunc (module, "llvm.x86.sse3.hsub.pd", ret_type, arg_types, 2);
7437 AddFunc (module, "llvm.x86.sse3.addsub.pd", ret_type, arg_types, 2);
7439 ret_type = type_to_simd_type (MONO_TYPE_R4);
7440 arg_types [0] = ret_type;
7441 arg_types [1] = ret_type;
7442 AddFunc (module, "llvm.x86.sse.min.ps", ret_type, arg_types, 2);
7443 AddFunc (module, "llvm.x86.sse.max.ps", ret_type, arg_types, 2);
7444 AddFunc (module, "llvm.x86.sse3.hadd.ps", ret_type, arg_types, 2);
7445 AddFunc (module, "llvm.x86.sse3.hsub.ps", ret_type, arg_types, 2);
7446 AddFunc (module, "llvm.x86.sse3.addsub.ps", ret_type, arg_types, 2);
7449 ret_type = type_to_simd_type (MONO_TYPE_I1);
7450 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
7451 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
7452 AddFunc (module, "llvm.x86.sse2.packsswb.128", ret_type, arg_types, 2);
7453 AddFunc (module, "llvm.x86.sse2.packuswb.128", ret_type, arg_types, 2);
7454 ret_type = type_to_simd_type (MONO_TYPE_I2);
7455 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
7456 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
7457 AddFunc (module, "llvm.x86.sse2.packssdw.128", ret_type, arg_types, 2);
7458 AddFunc (module, "llvm.x86.sse41.packusdw", ret_type, arg_types, 2);
7461 ret_type = type_to_simd_type (MONO_TYPE_R8);
7462 arg_types [0] = ret_type;
7463 arg_types [1] = ret_type;
7464 arg_types [2] = LLVMInt8Type ();
7465 AddFunc (module, "llvm.x86.sse2.cmp.pd", ret_type, arg_types, 3);
7466 ret_type = type_to_simd_type (MONO_TYPE_R4);
7467 arg_types [0] = ret_type;
7468 arg_types [1] = ret_type;
7469 arg_types [2] = LLVMInt8Type ();
7470 AddFunc (module, "llvm.x86.sse.cmp.ps", ret_type, arg_types, 3);
7472 /* Conversion ops */
7473 ret_type = type_to_simd_type (MONO_TYPE_R8);
7474 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
7475 AddFunc (module, "llvm.x86.sse2.cvtdq2pd", ret_type, arg_types, 1);
7476 ret_type = type_to_simd_type (MONO_TYPE_R4);
7477 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
7478 AddFunc (module, "llvm.x86.sse2.cvtdq2ps", ret_type, arg_types, 1);
7479 ret_type = type_to_simd_type (MONO_TYPE_I4);
7480 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7481 AddFunc (module, "llvm.x86.sse2.cvtpd2dq", ret_type, arg_types, 1);
7482 ret_type = type_to_simd_type (MONO_TYPE_I4);
7483 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7484 AddFunc (module, "llvm.x86.sse2.cvtps2dq", ret_type, arg_types, 1);
7485 ret_type = type_to_simd_type (MONO_TYPE_R4);
7486 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7487 AddFunc (module, "llvm.x86.sse2.cvtpd2ps", ret_type, arg_types, 1);
7488 ret_type = type_to_simd_type (MONO_TYPE_R8);
7489 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7490 AddFunc (module, "llvm.x86.sse2.cvtps2pd", ret_type, arg_types, 1);
7492 ret_type = type_to_simd_type (MONO_TYPE_I4);
7493 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
7494 AddFunc (module, "llvm.x86.sse2.cvttpd2dq", ret_type, arg_types, 1);
7495 ret_type = type_to_simd_type (MONO_TYPE_I4);
7496 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
7497 AddFunc (module, "llvm.x86.sse2.cvttps2dq", ret_type, arg_types, 1);
7500 ret_type = type_to_simd_type (MONO_TYPE_R8);
7501 arg_types [0] = ret_type;
7502 AddFunc (module, "llvm.x86.sse2.sqrt.pd", ret_type, arg_types, 1);
7503 ret_type = type_to_simd_type (MONO_TYPE_R4);
7504 arg_types [0] = ret_type;
7505 AddFunc (module, "llvm.x86.sse.sqrt.ps", ret_type, arg_types, 1);
7506 ret_type = type_to_simd_type (MONO_TYPE_R4);
7507 arg_types [0] = ret_type;
7508 AddFunc (module, "llvm.x86.sse.rsqrt.ps", ret_type, arg_types, 1);
7509 ret_type = type_to_simd_type (MONO_TYPE_R4);
7510 arg_types [0] = ret_type;
7511 AddFunc (module, "llvm.x86.sse.rcp.ps", ret_type, arg_types, 1);
7514 ret_type = type_to_simd_type (MONO_TYPE_I2);
7515 arg_types [0] = ret_type;
7516 arg_types [1] = LLVMInt32Type ();
7517 AddFunc (module, "llvm.x86.sse2.psrli.w", ret_type, arg_types, 2);
7518 AddFunc (module, "llvm.x86.sse2.psrai.w", ret_type, arg_types, 2);
7519 AddFunc (module, "llvm.x86.sse2.pslli.w", ret_type, arg_types, 2);
7520 ret_type = type_to_simd_type (MONO_TYPE_I4);
7521 arg_types [0] = ret_type;
7522 arg_types [1] = LLVMInt32Type ();
7523 AddFunc (module, "llvm.x86.sse2.psrli.d", ret_type, arg_types, 2);
7524 AddFunc (module, "llvm.x86.sse2.psrai.d", ret_type, arg_types, 2);
7525 AddFunc (module, "llvm.x86.sse2.pslli.d", ret_type, arg_types, 2);
7526 ret_type = type_to_simd_type (MONO_TYPE_I8);
7527 arg_types [0] = ret_type;
7528 arg_types [1] = LLVMInt32Type ();
7529 AddFunc (module, "llvm.x86.sse2.psrli.q", ret_type, arg_types, 2);
7530 AddFunc (module, "llvm.x86.sse2.pslli.q", ret_type, arg_types, 2);
7533 ret_type = LLVMInt32Type ();
7534 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
7535 AddFunc (module, "llvm.x86.sse2.pmovmskb.128", ret_type, arg_types, 1);
7538 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
7541 /* Load/Store intrinsics */
7543 LLVMTypeRef arg_types [5];
7547 for (i = 1; i <= 8; i *= 2) {
7548 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
7549 arg_types [1] = LLVMInt32Type ();
7550 arg_types [2] = LLVMInt1Type ();
7551 arg_types [3] = LLVMInt32Type ();
7552 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
7553 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
7555 arg_types [0] = LLVMIntType (i * 8);
7556 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
7557 arg_types [2] = LLVMInt32Type ();
7558 arg_types [3] = LLVMInt1Type ();
7559 arg_types [4] = LLVMInt32Type ();
7560 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
7561 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
7567 add_types (MonoLLVMModule *module)
7569 module->ptr_type = LLVMPointerType (sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
7573 mono_llvm_init (void)
7575 mono_native_tls_alloc (¤t_cfg_tls_id, NULL);
7579 init_jit_module (MonoDomain *domain)
7581 MonoJitICallInfo *info;
7582 MonoJitDomainInfo *dinfo;
7583 MonoLLVMModule *module;
7586 dinfo = domain_jit_info (domain);
7587 if (dinfo->llvm_module)
7590 mono_loader_lock ();
7592 if (dinfo->llvm_module) {
7593 mono_loader_unlock ();
7597 module = g_new0 (MonoLLVMModule, 1);
7599 name = g_strdup_printf ("mono-%s", domain->friendly_name);
7600 module->lmodule = LLVMModuleCreateWithName (name);
7601 module->context = LLVMGetGlobalContext ();
7603 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->lmodule), alloc_cb, emitted_cb, exception_cb, dlsym_cb, &module->ee);
7605 add_intrinsics (module->lmodule);
7608 module->llvm_types = g_hash_table_new (NULL, NULL);
7610 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
7612 LLVMAddGlobalMapping (module->ee, LLVMGetNamedFunction (module->lmodule, "llvm_resume_unwind_trampoline"), (void*)info->func);
7614 mono_memory_barrier ();
7616 dinfo->llvm_module = module;
7618 mono_loader_unlock ();
7622 mono_llvm_cleanup (void)
7624 MonoLLVMModule *module = &aot_module;
7626 if (module->lmodule)
7627 LLVMDisposeModule (module->lmodule);
7629 if (module->context)
7630 LLVMContextDispose (module->context);
7634 mono_llvm_free_domain_info (MonoDomain *domain)
7636 MonoJitDomainInfo *info = domain_jit_info (domain);
7637 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
7643 if (module->llvm_types)
7644 g_hash_table_destroy (module->llvm_types);
7646 mono_llvm_dispose_ee (module->mono_ee);
7648 if (module->bb_names) {
7649 for (i = 0; i < module->bb_names_len; ++i)
7650 g_free (module->bb_names [i]);
7651 g_free (module->bb_names);
7653 //LLVMDisposeModule (module->module);
7657 info->llvm_module = NULL;
7661 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, gboolean emit_dwarf, gboolean static_link, gboolean llvm_only)
7663 MonoLLVMModule *module = &aot_module;
7665 /* Delete previous module */
7666 if (module->plt_entries)
7667 g_hash_table_destroy (module->plt_entries);
7668 if (module->lmodule)
7669 LLVMDisposeModule (module->lmodule);
7671 memset (module, 0, sizeof (aot_module));
7673 module->lmodule = LLVMModuleCreateWithName ("aot");
7674 module->assembly = assembly;
7675 module->global_prefix = g_strdup (global_prefix);
7676 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
7677 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
7678 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
7679 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
7680 module->external_symbols = TRUE;
7681 module->emit_dwarf = emit_dwarf;
7682 module->static_link = static_link;
7683 module->llvm_only = llvm_only;
7684 /* The first few entries are reserved */
7685 module->max_got_offset = 16;
7686 module->context = LLVMContextCreate ();
7689 /* clang ignores our debug info because it has an invalid version */
7690 module->emit_dwarf = FALSE;
7692 add_intrinsics (module->lmodule);
7697 * We couldn't compute the type of the LLVM global representing the got because
7698 * its size is only known after all the methods have been emitted. So create
7699 * a dummy variable, and replace all uses it with the real got variable when
7700 * its size is known in mono_llvm_emit_aot_module ().
7703 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
7705 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
7706 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
7709 /* Add initialization array */
7711 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
7713 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
7714 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
7718 emit_init_icall_wrappers (module);
7720 emit_llvm_code_start (module);
7722 /* Add a dummy personality function */
7723 if (!use_debug_personality) {
7724 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
7725 LLVMSetLinkage (personality, LLVMExternalLinkage);
7726 mark_as_used (module, personality);
7729 /* Add a reference to the c++ exception we throw/catch */
7731 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
7732 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
7733 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
7734 mono_llvm_set_is_constant (module->sentinel_exception);
7737 module->llvm_types = g_hash_table_new (NULL, NULL);
7738 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
7739 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
7740 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
7741 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
7742 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
7743 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
7744 module->method_to_callers = g_hash_table_new (NULL, NULL);
7748 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
7751 LLVMValueRef res, *vals;
7753 vals = g_new0 (LLVMValueRef, nvalues);
7754 for (i = 0; i < nvalues; ++i)
7755 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
7756 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
7762 * mono_llvm_emit_aot_file_info:
7764 * Emit the MonoAotFileInfo structure.
7765 * Same as emit_aot_file_info () in aot-compiler.c.
7768 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
7770 MonoLLVMModule *module = &aot_module;
7772 /* Save these for later */
7773 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
7774 module->has_jitted_code = has_jitted_code;
7778 * mono_llvm_emit_aot_data:
7780 * Emit the binary data DATA pointed to by symbol SYMBOL.
7783 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
7785 MonoLLVMModule *module = &aot_module;
7789 type = LLVMArrayType (LLVMInt8Type (), data_len);
7790 d = LLVMAddGlobal (module->lmodule, type, symbol);
7791 LLVMSetVisibility (d, LLVMHiddenVisibility);
7792 LLVMSetLinkage (d, LLVMInternalLinkage);
7793 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
7794 mono_llvm_set_is_constant (d);
7797 /* Add a reference to a global defined in JITted code */
7799 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
7804 s = g_strdup_printf ("%s%s", module->global_prefix, name);
7805 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
7811 emit_aot_file_info (MonoLLVMModule *module)
7813 LLVMTypeRef file_info_type;
7814 LLVMTypeRef *eltypes, eltype;
7815 LLVMValueRef info_var;
7816 LLVMValueRef *fields;
7817 int i, nfields, tindex;
7818 MonoAotFileInfo *info;
7819 LLVMModuleRef lmodule = module->lmodule;
7821 info = &module->aot_info;
7823 /* Create an LLVM type to represent MonoAotFileInfo */
7824 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 15 + 5;
7825 eltypes = g_new (LLVMTypeRef, nfields);
7827 eltypes [tindex ++] = LLVMInt32Type ();
7828 eltypes [tindex ++] = LLVMInt32Type ();
7830 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
7831 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
7833 for (i = 0; i < 15; ++i)
7834 eltypes [tindex ++] = LLVMInt32Type ();
7836 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
7837 for (i = 0; i < 4; ++i)
7838 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
7839 g_assert (tindex == nfields);
7840 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
7841 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
7843 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
7844 if (module->static_link) {
7845 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
7846 LLVMSetLinkage (info_var, LLVMInternalLinkage);
7848 fields = g_new (LLVMValueRef, nfields);
7850 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
7851 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
7855 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
7856 * for symbols defined in the .s file emitted by the aot compiler.
7858 eltype = eltypes [tindex];
7859 if (module->llvm_only)
7860 fields [tindex ++] = LLVMConstNull (eltype);
7862 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
7863 fields [tindex ++] = module->got_var;
7864 /* llc defines this directly */
7865 if (!module->llvm_only) {
7866 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
7867 fields [tindex ++] = LLVMConstNull (eltype);
7868 fields [tindex ++] = LLVMConstNull (eltype);
7870 fields [tindex ++] = LLVMConstNull (eltype);
7871 fields [tindex ++] = module->get_method;
7872 fields [tindex ++] = module->get_unbox_tramp;
7874 if (module->has_jitted_code) {
7875 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
7876 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
7878 fields [tindex ++] = LLVMConstNull (eltype);
7879 fields [tindex ++] = LLVMConstNull (eltype);
7881 if (!module->llvm_only)
7882 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
7884 fields [tindex ++] = LLVMConstNull (eltype);
7885 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
7886 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
7887 fields [tindex ++] = LLVMConstNull (eltype);
7889 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
7890 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
7891 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
7892 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
7893 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
7894 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
7895 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
7896 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
7897 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
7898 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
7900 /* Not needed (mem_end) */
7901 fields [tindex ++] = LLVMConstNull (eltype);
7902 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
7903 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
7904 if (info->trampoline_size [0]) {
7905 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
7906 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
7907 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_thunks");
7908 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
7910 fields [tindex ++] = LLVMConstNull (eltype);
7911 fields [tindex ++] = LLVMConstNull (eltype);
7912 fields [tindex ++] = LLVMConstNull (eltype);
7913 fields [tindex ++] = LLVMConstNull (eltype);
7915 if (module->static_link && !module->llvm_only)
7916 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
7918 fields [tindex ++] = LLVMConstNull (eltype);
7919 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
7920 if (!module->llvm_only) {
7921 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
7922 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
7923 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
7924 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
7925 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
7926 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
7928 fields [tindex ++] = LLVMConstNull (eltype);
7929 fields [tindex ++] = LLVMConstNull (eltype);
7930 fields [tindex ++] = LLVMConstNull (eltype);
7931 fields [tindex ++] = LLVMConstNull (eltype);
7932 fields [tindex ++] = LLVMConstNull (eltype);
7933 fields [tindex ++] = LLVMConstNull (eltype);
7936 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
7937 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
7940 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
7941 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
7942 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
7943 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
7944 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
7945 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
7946 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
7947 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
7948 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
7949 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
7950 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
7951 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
7952 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
7953 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
7954 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
7956 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
7957 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
7958 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
7959 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
7960 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
7961 g_assert (tindex == nfields);
7963 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
7965 if (module->static_link) {
7969 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
7970 /* Get rid of characters which cannot occur in symbols */
7972 for (p = s; *p; ++p) {
7973 if (!(isalnum (*p) || *p == '_'))
7976 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
7978 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
7979 LLVMSetLinkage (var, LLVMExternalLinkage);
7984 * Emit the aot module into the LLVM bitcode file FILENAME.
7987 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
7989 LLVMTypeRef got_type, inited_type;
7990 LLVMValueRef real_got, real_inited;
7991 MonoLLVMModule *module = &aot_module;
7993 emit_llvm_code_end (module);
7996 * Create the real got variable and replace all uses of the dummy variable with
7999 got_type = LLVMArrayType (module->ptr_type, module->max_got_offset + 1);
8000 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
8001 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
8002 if (module->external_symbols) {
8003 LLVMSetLinkage (real_got, LLVMExternalLinkage);
8004 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
8006 LLVMSetLinkage (real_got, LLVMInternalLinkage);
8008 mono_llvm_replace_uses_of (module->got_var, real_got);
8010 mark_as_used (&aot_module, real_got);
8012 /* Delete the dummy got so it doesn't become a global */
8013 LLVMDeleteGlobal (module->got_var);
8014 module->got_var = real_got;
8017 * Same for the init_var
8019 if (module->llvm_only) {
8020 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
8021 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
8022 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
8023 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
8024 mono_llvm_replace_uses_of (module->inited_var, real_inited);
8025 LLVMDeleteGlobal (module->inited_var);
8028 if (module->llvm_only) {
8029 emit_get_method (&aot_module);
8030 emit_get_unbox_tramp (&aot_module);
8033 emit_llvm_used (&aot_module);
8034 emit_dbg_info (&aot_module, filename, cu_name);
8035 emit_aot_file_info (&aot_module);
8038 * Replace GOT entries for directly callable methods with the methods themselves.
8039 * It would be easier to implement this by predefining all methods before compiling
8040 * their bodies, but that couldn't handle the case when a method fails to compile
8043 if (module->llvm_only) {
8044 GHashTableIter iter;
8046 GSList *callers, *l;
8048 g_hash_table_iter_init (&iter, module->method_to_callers);
8049 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
8050 LLVMValueRef lmethod;
8052 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
8054 for (l = callers; l; l = l->next) {
8055 LLVMValueRef caller = (LLVMValueRef)l->data;
8057 mono_llvm_replace_uses_of (caller, lmethod);
8063 /* Replace PLT entries for directly callable methods with the methods themselves */
8065 GHashTableIter iter;
8067 LLVMValueRef callee;
8069 g_hash_table_iter_init (&iter, module->plt_entries_ji);
8070 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
8071 if (mono_aot_is_direct_callable (ji)) {
8072 LLVMValueRef lmethod;
8074 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
8075 /* The types might not match because the caller might pass an rgctx */
8076 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
8077 mono_llvm_replace_uses_of (callee, lmethod);
8078 mono_aot_mark_unused_llvm_plt_entry (ji);
8088 if (LLVMVerifyModule (module->module, LLVMReturnStatusAction, &verifier_err)) {
8089 g_assert_not_reached ();
8094 LLVMWriteBitcodeToFile (module->lmodule, filename);
8099 md_string (const char *s)
8101 return LLVMMDString (s, strlen (s));
8104 /* Debugging support */
8107 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
8109 LLVMModuleRef lmodule = module->lmodule;
8110 LLVMValueRef args [16], cu_args [16], cu, ver;
8112 char *build_info, *s, *dir;
8115 * This can only be enabled when LLVM code is emitted into a separate object
8116 * file, since the AOT compiler also emits dwarf info,
8117 * and the abbrev indexes will not be correct since llvm has added its own
8120 if (!module->emit_dwarf)
8124 * Emit dwarf info in the form of LLVM metadata. There is some
8125 * out-of-date documentation at:
8126 * http://llvm.org/docs/SourceLevelDebugging.html
8127 * but most of this was gathered from the llvm and
8132 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_TAG_compile_unit, FALSE);
8133 /* CU name/compilation dir */
8134 dir = g_path_get_dirname (filename);
8135 args [0] = LLVMMDString (cu_name, strlen (cu_name));
8136 args [1] = LLVMMDString (dir, strlen (dir));
8137 cu_args [n_cuargs ++] = LLVMMDNode (args, 2);
8140 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_LANG_C99, FALSE);
8142 build_info = mono_get_runtime_build_info ();
8143 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
8144 cu_args [n_cuargs ++] = LLVMMDString (s, strlen (s));
8145 g_free (build_info);
8147 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8149 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
8150 /* Runtime version */
8151 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8153 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8154 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8156 if (module->subprogram_mds) {
8160 mds = g_new0 (LLVMValueRef, module->subprogram_mds->len);
8161 for (i = 0; i < module->subprogram_mds->len; ++i)
8162 mds [i] = (LLVMValueRef)g_ptr_array_index (module->subprogram_mds, i);
8163 cu_args [n_cuargs ++] = LLVMMDNode (mds, module->subprogram_mds->len);
8165 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8168 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8169 /* Imported modules */
8170 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8172 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
8173 /* DebugEmissionKind = FullDebug */
8174 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8175 cu = LLVMMDNode (cu_args, n_cuargs);
8176 LLVMAddNamedMetadataOperand (lmodule, "llvm.dbg.cu", cu);
8178 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8179 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
8180 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8181 ver = LLVMMDNode (args, 3);
8182 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8184 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8185 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
8186 args [2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8187 ver = LLVMMDNode (args, 3);
8188 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8192 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
8194 MonoLLVMModule *module = ctx->module;
8195 MonoDebugMethodInfo *minfo = ctx->minfo;
8196 char *source_file, *dir, *filename;
8197 LLVMValueRef md, args [16], ctx_args [16], md_args [64], type_args [16], ctx_md, type_md;
8198 MonoSymSeqPoint *sym_seq_points;
8204 mono_debug_symfile_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
8206 source_file = g_strdup ("<unknown>");
8207 dir = g_path_get_dirname (source_file);
8208 filename = g_path_get_basename (source_file);
8210 ctx_args [0] = LLVMConstInt (LLVMInt32Type (), 0x29, FALSE);
8211 args [0] = md_string (filename);
8212 args [1] = md_string (dir);
8213 ctx_args [1] = LLVMMDNode (args, 2);
8214 ctx_md = LLVMMDNode (ctx_args, 2);
8216 type_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subroutine_type, FALSE);
8217 type_args [1] = NULL;
8218 type_args [2] = NULL;
8219 type_args [3] = LLVMMDString ("", 0);
8220 type_args [4] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8221 type_args [5] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8222 type_args [6] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8223 type_args [7] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8224 type_args [8] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8225 type_args [9] = NULL;
8226 type_args [10] = NULL;
8227 type_args [11] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8228 type_args [12] = NULL;
8229 type_args [13] = NULL;
8230 type_args [14] = NULL;
8231 type_md = LLVMMDNode (type_args, 14);
8233 /* http://llvm.org/docs/SourceLevelDebugging.html#subprogram-descriptors */
8234 md_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subprogram, FALSE);
8235 /* Source directory + file pair */
8236 args [0] = md_string (filename);
8237 args [1] = md_string (dir);
8238 md_args [1] = LLVMMDNode (args ,2);
8239 md_args [2] = ctx_md;
8240 md_args [3] = md_string (cfg->method->name);
8241 md_args [4] = md_string (name);
8242 md_args [5] = md_string (name);
8245 md_args [6] = LLVMConstInt (LLVMInt32Type (), sym_seq_points [0].line, FALSE);
8247 md_args [6] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8249 md_args [7] = type_md;
8251 md_args [8] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8253 md_args [9] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8255 md_args [10] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8256 /* Index into a virtual function */
8257 md_args [11] = NULL;
8258 md_args [12] = NULL;
8260 md_args [13] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8262 md_args [14] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8263 /* Pointer to LLVM function */
8264 md_args [15] = method;
8265 /* Function template parameter */
8266 md_args [16] = NULL;
8267 /* Function declaration descriptor */
8268 md_args [17] = NULL;
8269 /* List of function variables */
8270 md_args [18] = LLVMMDNode (args, 0);
8272 md_args [19] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8273 md = LLVMMDNode (md_args, 20);
8275 if (!module->subprogram_mds)
8276 module->subprogram_mds = g_ptr_array_new ();
8277 g_ptr_array_add (module->subprogram_mds, md);
8281 g_free (source_file);
8282 g_free (sym_seq_points);
8288 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
8290 MonoCompile *cfg = ctx->cfg;
8292 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
8293 MonoDebugSourceLocation *loc;
8294 LLVMValueRef loc_md, md_args [16];
8297 loc = mono_debug_symfile_lookup_location (ctx->minfo, cil_code - cfg->header->code);
8301 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->row, FALSE);
8302 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->column, FALSE);
8303 md_args [nmd_args ++] = ctx->dbg_md;
8304 md_args [nmd_args ++] = NULL;
8305 loc_md = LLVMMDNode (md_args, nmd_args);
8306 LLVMSetCurrentDebugLocation (builder, loc_md);
8307 mono_debug_symfile_free_location (loc);
8313 default_mono_llvm_unhandled_exception (void)
8315 MonoJitTlsData *jit_tls = mono_get_jit_tls ();
8316 MonoObject *target = mono_gchandle_get_target (jit_tls->thrown_exc);
8318 mono_unhandled_exception (target);
8319 exit (mono_environment_exitcode_get ());
8324 - Emit LLVM IR from the mono IR using the LLVM C API.
8325 - The original arch specific code remains, so we can fall back to it if we run
8326 into something we can't handle.
8330 A partial list of issues:
8331 - Handling of opcodes which can throw exceptions.
8333 In the mono JIT, these are implemented using code like this:
8340 push throw_pos - method
8341 call <exception trampoline>
8343 The problematic part is push throw_pos - method, which cannot be represented
8344 in the LLVM IR, since it does not support label values.
8345 -> this can be implemented in AOT mode using inline asm + labels, but cannot
8346 be implemented in JIT mode ?
8347 -> a possible but slower implementation would use the normal exception
8348 throwing code but it would need to control the placement of the throw code
8349 (it needs to be exactly after the compare+branch).
8350 -> perhaps add a PC offset intrinsics ?
8352 - efficient implementation of .ovf opcodes.
8354 These are currently implemented as:
8355 <ins which sets the condition codes>
8358 Some overflow opcodes are now supported by LLVM SVN.
8360 - exception handling, unwinding.
8361 - SSA is disabled for methods with exception handlers
8362 - How to obtain unwind info for LLVM compiled methods ?
8363 -> this is now solved by converting the unwind info generated by LLVM
8365 - LLVM uses the c++ exception handling framework, while we use our home grown
8366 code, and couldn't use the c++ one:
8367 - its not supported under VC++, other exotic platforms.
8368 - it might be impossible to support filter clauses with it.
8372 The trampolines need a predictable call sequence, since they need to disasm
8373 the calling code to obtain register numbers / offsets.
8375 LLVM currently generates this code in non-JIT mode:
8376 mov -0x98(%rax),%eax
8378 Here, the vtable pointer is lost.
8379 -> solution: use one vtable trampoline per class.
8381 - passing/receiving the IMT pointer/RGCTX.
8382 -> solution: pass them as normal arguments ?
8386 LLVM does not allow the specification of argument registers etc. This means
8387 that all calls are made according to the platform ABI.
8389 - passing/receiving vtypes.
8391 Vtypes passed/received in registers are handled by the front end by using
8392 a signature with scalar arguments, and loading the parts of the vtype into those
8395 Vtypes passed on the stack are handled using the 'byval' attribute.
8399 Supported though alloca, we need to emit the load/store code.
8403 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
8404 typed registers, so we have to keep track of the precise LLVM type of each vreg.
8405 This is made easier because the IR is already in SSA form.
8406 An additional problem is that our IR is not consistent with types, i.e. i32/ia64
8407 types are frequently used incorrectly.
8412 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
8413 it with the file containing the methods emitted by the JIT and the AOT data
8417 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
8418 * - each bblock should end with a branch
8419 * - setting the return value, making cfg->ret non-volatile
8420 * - avoid some transformations in the JIT which make it harder for us to generate
8422 * - use pointer types to help optimizations.