2 * mini-llvm.c: llvm "Backend" for the mono JIT
4 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
5 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
6 * Licensed under the MIT license. See LICENSE file in the project root for full license information.
10 #include <mono/metadata/debug-helpers.h>
11 #include <mono/metadata/debug-mono-symfile.h>
12 #include <mono/metadata/mempool-internals.h>
13 #include <mono/metadata/environment.h>
14 #include <mono/metadata/object-internals.h>
15 #include <mono/metadata/abi-details.h>
16 #include <mono/utils/mono-tls.h>
17 #include <mono/utils/mono-dl.h>
18 #include <mono/utils/mono-time.h>
19 #include <mono/utils/freebsd-dwarf.h>
21 #ifndef __STDC_LIMIT_MACROS
22 #define __STDC_LIMIT_MACROS
24 #ifndef __STDC_CONSTANT_MACROS
25 #define __STDC_CONSTANT_MACROS
28 #include "llvm-c/BitWriter.h"
29 #include "llvm-c/Analysis.h"
31 #include "mini-llvm-cpp.h"
33 #include "aot-compiler.h"
34 #include "mini-llvm.h"
39 extern void *memset(void *, int, size_t);
40 void bzero (void *to, size_t count) { memset (to, 0, count); }
44 #if LLVM_API_VERSION < 4
45 #error "The version of the mono llvm repository is too old."
48 #define ALIGN_PTR_TO(ptr,align) (gpointer)((((gssize)(ptr)) + (align - 1)) & (~(align - 1)))
51 * Information associated by mono with LLVM modules.
54 LLVMModuleRef lmodule;
55 LLVMValueRef throw_icall, rethrow, match_exc, throw_corlib_exception, resume_eh;
56 GHashTable *llvm_types;
58 const char *got_symbol;
59 const char *get_method_symbol;
60 const char *get_unbox_tramp_symbol;
61 GHashTable *plt_entries;
62 GHashTable *plt_entries_ji;
63 GHashTable *method_to_lmethod;
64 GHashTable *direct_callables;
69 GPtrArray *subprogram_mds;
71 LLVMExecutionEngineRef ee;
72 gboolean external_symbols;
75 LLVMValueRef personality;
78 MonoAssembly *assembly;
80 MonoAotFileInfo aot_info;
81 const char *jit_got_symbol;
82 const char *eh_frame_symbol;
83 LLVMValueRef get_method, get_unbox_tramp;
84 LLVMValueRef init_method, init_method_gshared_mrgctx, init_method_gshared_this, init_method_gshared_vtable;
85 LLVMValueRef code_start, code_end;
86 LLVMValueRef inited_var;
87 int max_inited_idx, max_method_idx;
88 gboolean has_jitted_code;
91 GHashTable *idx_to_lmethod;
92 GHashTable *idx_to_unbox_tramp;
93 /* Maps a MonoMethod to LLVM instructions representing it */
94 GHashTable *method_to_callers;
95 LLVMContextRef context;
96 LLVMValueRef sentinel_exception;
97 void *di_builder, *cu;
98 GHashTable *objc_selector_to_var;
102 * Information associated by the backend with mono basic blocks.
105 LLVMBasicBlockRef bblock, end_bblock;
106 LLVMValueRef finally_ind;
107 gboolean added, invoke_target;
109 * If this bblock is the start of a finally clause, this is a list of bblocks it
110 * needs to branch to in ENDFINALLY.
112 GSList *call_handler_return_bbs;
114 * If this bblock is the start of a finally clause, this is the bblock that
115 * CALL_HANDLER needs to branch to.
117 LLVMBasicBlockRef call_handler_target_bb;
118 /* The list of switch statements generated by ENDFINALLY instructions */
119 GSList *endfinally_switch_ins_list;
124 * Structure containing emit state
127 MonoMemPool *mempool;
129 /* Maps method names to the corresponding LLVMValueRef */
130 GHashTable *emitted_method_decls;
133 LLVMValueRef lmethod;
134 MonoLLVMModule *module;
135 LLVMModuleRef lmodule;
137 int sindex, default_index, ex_index;
138 LLVMBuilderRef builder;
139 LLVMValueRef *values, *addresses;
140 MonoType **vreg_cli_types;
142 MonoMethodSignature *sig;
144 GHashTable *region_to_handler;
145 GHashTable *clause_to_handler;
146 LLVMBuilderRef alloca_builder;
147 LLVMValueRef last_alloca;
148 LLVMValueRef rgctx_arg;
149 LLVMValueRef this_arg;
150 LLVMTypeRef *vreg_types;
151 LLVMTypeRef method_type;
152 LLVMBasicBlockRef init_bb, inited_bb;
154 gboolean *unreachable;
156 gboolean has_got_access;
157 gboolean is_linkonce;
158 int this_arg_pindex, rgctx_arg_pindex;
159 LLVMValueRef imt_rgctx_loc;
160 GHashTable *llvm_types;
162 MonoDebugMethodInfo *minfo;
164 /* For every clause, the clauses it is nested in */
167 GHashTable *exc_meta;
168 GHashTable *method_to_callers;
169 GPtrArray *phi_values;
170 GPtrArray *bblock_list;
172 GHashTable *jit_callees;
178 MonoBasicBlock *in_bb;
183 * Instruction metadata
184 * This is the same as ins_info, but LREG != IREG.
192 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
193 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
200 /* keep in sync with the enum in mini.h */
203 #include "mini-ops.h"
208 #if SIZEOF_VOID_P == 4
209 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
211 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
214 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
217 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
219 #define TRACE_FAILURE(msg)
223 #define IS_TARGET_X86 1
225 #define IS_TARGET_X86 0
229 #define IS_TARGET_AMD64 1
231 #define IS_TARGET_AMD64 0
234 #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm)
236 static LLVMIntPredicate cond_to_llvm_cond [] = {
249 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
262 static MonoNativeTlsKey current_cfg_tls_id;
264 static MonoLLVMModule aot_module;
266 static GHashTable *intrins_id_to_name;
267 static GHashTable *intrins_name_to_id;
269 static void init_jit_module (MonoDomain *domain);
271 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
272 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
273 static void emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name);
274 static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp);
275 static LLVMValueRef get_intrinsic (EmitContext *ctx, const char *name);
276 static void decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame);
279 set_failure (EmitContext *ctx, const char *message)
281 TRACE_FAILURE (reason);
282 ctx->cfg->exception_message = g_strdup (message);
283 ctx->cfg->disable_llvm = TRUE;
289 * The LLVM type with width == sizeof (gpointer)
294 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
300 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
306 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
312 * Return the size of the LLVM representation of the vtype T.
315 get_vtype_size (MonoType *t)
319 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
321 /* LLVMArgAsIArgs depends on this since it stores whole words */
322 while (size < 2 * sizeof (gpointer) && mono_is_power_of_two (size) == -1)
329 * simd_class_to_llvm_type:
331 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
334 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
336 if (!strcmp (klass->name, "Vector2d")) {
337 return LLVMVectorType (LLVMDoubleType (), 2);
338 } else if (!strcmp (klass->name, "Vector2l")) {
339 return LLVMVectorType (LLVMInt64Type (), 2);
340 } else if (!strcmp (klass->name, "Vector2ul")) {
341 return LLVMVectorType (LLVMInt64Type (), 2);
342 } else if (!strcmp (klass->name, "Vector4i")) {
343 return LLVMVectorType (LLVMInt32Type (), 4);
344 } else if (!strcmp (klass->name, "Vector4ui")) {
345 return LLVMVectorType (LLVMInt32Type (), 4);
346 } else if (!strcmp (klass->name, "Vector4f")) {
347 return LLVMVectorType (LLVMFloatType (), 4);
348 } else if (!strcmp (klass->name, "Vector8s")) {
349 return LLVMVectorType (LLVMInt16Type (), 8);
350 } else if (!strcmp (klass->name, "Vector8us")) {
351 return LLVMVectorType (LLVMInt16Type (), 8);
352 } else if (!strcmp (klass->name, "Vector16sb")) {
353 return LLVMVectorType (LLVMInt8Type (), 16);
354 } else if (!strcmp (klass->name, "Vector16b")) {
355 return LLVMVectorType (LLVMInt8Type (), 16);
357 printf ("%s\n", klass->name);
363 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
364 static inline G_GNUC_UNUSED LLVMTypeRef
365 type_to_simd_type (int type)
369 return LLVMVectorType (LLVMInt8Type (), 16);
371 return LLVMVectorType (LLVMInt16Type (), 8);
373 return LLVMVectorType (LLVMInt32Type (), 4);
375 return LLVMVectorType (LLVMInt64Type (), 2);
377 return LLVMVectorType (LLVMDoubleType (), 2);
379 return LLVMVectorType (LLVMFloatType (), 4);
381 g_assert_not_reached ();
387 create_llvm_type_for_type (MonoLLVMModule *module, MonoClass *klass)
389 int i, size, nfields, esize;
390 LLVMTypeRef *eltypes;
395 t = &klass->byval_arg;
397 if (mini_type_is_hfa (t, &nfields, &esize)) {
399 * This is needed on arm64 where HFAs are returned in
403 eltypes = g_new (LLVMTypeRef, size);
404 for (i = 0; i < size; ++i)
405 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
407 size = get_vtype_size (t);
409 eltypes = g_new (LLVMTypeRef, size);
410 for (i = 0; i < size; ++i)
411 eltypes [i] = LLVMInt8Type ();
414 name = mono_type_full_name (&klass->byval_arg);
415 ltype = LLVMStructCreateNamed (module->context, name);
416 LLVMStructSetBody (ltype, eltypes, size, FALSE);
426 * Return the LLVM type corresponding to T.
429 type_to_llvm_type (EmitContext *ctx, MonoType *t)
431 t = mini_get_underlying_type (t);
435 return LLVMVoidType ();
437 return LLVMInt8Type ();
439 return LLVMInt16Type ();
441 return LLVMInt32Type ();
443 return LLVMInt8Type ();
445 return LLVMInt16Type ();
447 return LLVMInt32Type ();
448 case MONO_TYPE_BOOLEAN:
449 return LLVMInt8Type ();
452 return LLVMInt64Type ();
454 return LLVMInt16Type ();
456 return LLVMFloatType ();
458 return LLVMDoubleType ();
461 return IntPtrType ();
462 case MONO_TYPE_OBJECT:
463 case MONO_TYPE_CLASS:
464 case MONO_TYPE_ARRAY:
465 case MONO_TYPE_SZARRAY:
466 case MONO_TYPE_STRING:
468 return ObjRefType ();
471 /* Because of generic sharing */
472 return ObjRefType ();
473 case MONO_TYPE_GENERICINST:
474 if (!mono_type_generic_inst_is_valuetype (t))
475 return ObjRefType ();
477 case MONO_TYPE_VALUETYPE:
478 case MONO_TYPE_TYPEDBYREF: {
482 klass = mono_class_from_mono_type (t);
484 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
485 return simd_class_to_llvm_type (ctx, klass);
488 return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
490 ltype = (LLVMTypeRef)g_hash_table_lookup (ctx->module->llvm_types, klass);
492 ltype = create_llvm_type_for_type (ctx->module, klass);
493 g_hash_table_insert (ctx->module->llvm_types, klass, ltype);
499 printf ("X: %d\n", t->type);
500 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
501 ctx->cfg->disable_llvm = TRUE;
509 * Return whenever T is an unsigned int type.
512 type_is_unsigned (EmitContext *ctx, MonoType *t)
514 t = mini_get_underlying_type (t);
530 * type_to_llvm_arg_type:
532 * Same as type_to_llvm_type, but treat i8/i16 as i32.
535 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
537 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
539 if (ctx->cfg->llvm_only)
543 * This works on all abis except arm64/ios which passes multiple
544 * arguments in one stack slot.
547 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
549 * LLVM generates code which only sets the lower bits, while JITted
550 * code expects all the bits to be set.
552 ptype = LLVMInt32Type ();
560 * llvm_type_to_stack_type:
562 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
565 static G_GNUC_UNUSED LLVMTypeRef
566 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
570 if (type == LLVMInt8Type ())
571 return LLVMInt32Type ();
572 else if (type == LLVMInt16Type ())
573 return LLVMInt32Type ();
574 else if (!cfg->r4fp && type == LLVMFloatType ())
575 return LLVMDoubleType ();
581 * regtype_to_llvm_type:
583 * Return the LLVM type corresponding to the regtype C used in instruction
587 regtype_to_llvm_type (char c)
591 return LLVMInt32Type ();
593 return LLVMInt64Type ();
595 return LLVMDoubleType ();
604 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
607 op_to_llvm_type (int opcode)
612 return LLVMInt8Type ();
615 return LLVMInt8Type ();
618 return LLVMInt16Type ();
621 return LLVMInt16Type ();
624 return LLVMInt32Type ();
627 return LLVMInt32Type ();
629 return LLVMInt64Type ();
631 return LLVMFloatType ();
633 return LLVMDoubleType ();
635 return LLVMInt64Type ();
637 return LLVMInt32Type ();
639 return LLVMInt64Type ();
644 return LLVMInt8Type ();
649 return LLVMInt16Type ();
651 return LLVMInt32Type ();
654 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
661 return LLVMInt32Type ();
668 return LLVMInt64Type ();
670 printf ("%s\n", mono_inst_name (opcode));
671 g_assert_not_reached ();
676 #define CLAUSE_START(clause) ((clause)->try_offset)
677 #define CLAUSE_END(clause) (((clause))->try_offset + ((clause))->try_len)
680 * load_store_to_llvm_type:
682 * Return the size/sign/zero extension corresponding to the load/store opcode
686 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
692 case OP_LOADI1_MEMBASE:
693 case OP_STOREI1_MEMBASE_REG:
694 case OP_STOREI1_MEMBASE_IMM:
695 case OP_ATOMIC_LOAD_I1:
696 case OP_ATOMIC_STORE_I1:
699 return LLVMInt8Type ();
700 case OP_LOADU1_MEMBASE:
702 case OP_ATOMIC_LOAD_U1:
703 case OP_ATOMIC_STORE_U1:
706 return LLVMInt8Type ();
707 case OP_LOADI2_MEMBASE:
708 case OP_STOREI2_MEMBASE_REG:
709 case OP_STOREI2_MEMBASE_IMM:
710 case OP_ATOMIC_LOAD_I2:
711 case OP_ATOMIC_STORE_I2:
714 return LLVMInt16Type ();
715 case OP_LOADU2_MEMBASE:
717 case OP_ATOMIC_LOAD_U2:
718 case OP_ATOMIC_STORE_U2:
721 return LLVMInt16Type ();
722 case OP_LOADI4_MEMBASE:
723 case OP_LOADU4_MEMBASE:
726 case OP_STOREI4_MEMBASE_REG:
727 case OP_STOREI4_MEMBASE_IMM:
728 case OP_ATOMIC_LOAD_I4:
729 case OP_ATOMIC_STORE_I4:
730 case OP_ATOMIC_LOAD_U4:
731 case OP_ATOMIC_STORE_U4:
733 return LLVMInt32Type ();
734 case OP_LOADI8_MEMBASE:
736 case OP_STOREI8_MEMBASE_REG:
737 case OP_STOREI8_MEMBASE_IMM:
738 case OP_ATOMIC_LOAD_I8:
739 case OP_ATOMIC_STORE_I8:
740 case OP_ATOMIC_LOAD_U8:
741 case OP_ATOMIC_STORE_U8:
743 return LLVMInt64Type ();
744 case OP_LOADR4_MEMBASE:
745 case OP_STORER4_MEMBASE_REG:
746 case OP_ATOMIC_LOAD_R4:
747 case OP_ATOMIC_STORE_R4:
749 return LLVMFloatType ();
750 case OP_LOADR8_MEMBASE:
751 case OP_STORER8_MEMBASE_REG:
752 case OP_ATOMIC_LOAD_R8:
753 case OP_ATOMIC_STORE_R8:
755 return LLVMDoubleType ();
756 case OP_LOAD_MEMBASE:
758 case OP_STORE_MEMBASE_REG:
759 case OP_STORE_MEMBASE_IMM:
760 *size = sizeof (gpointer);
761 return IntPtrType ();
763 g_assert_not_reached ();
771 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
774 ovf_op_to_intrins (int opcode)
778 return "llvm.sadd.with.overflow.i32";
780 return "llvm.uadd.with.overflow.i32";
782 return "llvm.ssub.with.overflow.i32";
784 return "llvm.usub.with.overflow.i32";
786 return "llvm.smul.with.overflow.i32";
788 return "llvm.umul.with.overflow.i32";
790 return "llvm.sadd.with.overflow.i64";
792 return "llvm.uadd.with.overflow.i64";
794 return "llvm.ssub.with.overflow.i64";
796 return "llvm.usub.with.overflow.i64";
798 return "llvm.smul.with.overflow.i64";
800 return "llvm.umul.with.overflow.i64";
802 g_assert_not_reached ();
808 simd_op_to_intrins (int opcode)
811 #if defined(TARGET_X86) || defined(TARGET_AMD64)
813 return "llvm.x86.sse2.min.pd";
815 return "llvm.x86.sse.min.ps";
817 return "llvm.x86.sse2.max.pd";
819 return "llvm.x86.sse.max.ps";
821 return "llvm.x86.sse3.hadd.pd";
823 return "llvm.x86.sse3.hadd.ps";
825 return "llvm.x86.sse3.hsub.pd";
827 return "llvm.x86.sse3.hsub.ps";
829 return "llvm.x86.sse3.addsub.ps";
831 return "llvm.x86.sse3.addsub.pd";
832 case OP_EXTRACT_MASK:
833 return "llvm.x86.sse2.pmovmskb.128";
836 return "llvm.x86.sse2.psrli.w";
839 return "llvm.x86.sse2.psrli.d";
842 return "llvm.x86.sse2.psrli.q";
845 return "llvm.x86.sse2.pslli.w";
848 return "llvm.x86.sse2.pslli.d";
851 return "llvm.x86.sse2.pslli.q";
854 return "llvm.x86.sse2.psrai.w";
857 return "llvm.x86.sse2.psrai.d";
859 return "llvm.x86.sse2.padds.b";
861 return "llvm.x86.sse2.padds.w";
863 return "llvm.x86.sse2.psubs.b";
865 return "llvm.x86.sse2.psubs.w";
866 case OP_PADDB_SAT_UN:
867 return "llvm.x86.sse2.paddus.b";
868 case OP_PADDW_SAT_UN:
869 return "llvm.x86.sse2.paddus.w";
870 case OP_PSUBB_SAT_UN:
871 return "llvm.x86.sse2.psubus.b";
872 case OP_PSUBW_SAT_UN:
873 return "llvm.x86.sse2.psubus.w";
875 return "llvm.x86.sse2.pavg.b";
877 return "llvm.x86.sse2.pavg.w";
879 return "llvm.x86.sse.sqrt.ps";
881 return "llvm.x86.sse2.sqrt.pd";
883 return "llvm.x86.sse.rsqrt.ps";
885 return "llvm.x86.sse.rcp.ps";
887 return "llvm.x86.sse2.cvtdq2pd";
889 return "llvm.x86.sse2.cvtdq2ps";
891 return "llvm.x86.sse2.cvtpd2dq";
893 return "llvm.x86.sse2.cvtps2dq";
895 return "llvm.x86.sse2.cvtpd2ps";
897 return "llvm.x86.sse2.cvtps2pd";
899 return "llvm.x86.sse2.cvttpd2dq";
901 return "llvm.x86.sse2.cvttps2dq";
903 return "llvm.x86.sse2.packsswb.128";
905 return "llvm.x86.sse2.packssdw.128";
907 return "llvm.x86.sse2.packuswb.128";
909 return "llvm.x86.sse41.packusdw";
911 return "llvm.x86.sse2.pmulh.w";
912 case OP_PMULW_HIGH_UN:
913 return "llvm.x86.sse2.pmulhu.w";
916 g_assert_not_reached ();
922 simd_op_to_llvm_type (int opcode)
924 #if defined(TARGET_X86) || defined(TARGET_AMD64)
928 return type_to_simd_type (MONO_TYPE_R8);
931 return type_to_simd_type (MONO_TYPE_I8);
934 return type_to_simd_type (MONO_TYPE_I4);
939 return type_to_simd_type (MONO_TYPE_I2);
943 return type_to_simd_type (MONO_TYPE_I1);
945 return type_to_simd_type (MONO_TYPE_R4);
948 return type_to_simd_type (MONO_TYPE_I4);
952 return type_to_simd_type (MONO_TYPE_R8);
956 return type_to_simd_type (MONO_TYPE_R4);
957 case OP_EXTRACT_MASK:
958 return type_to_simd_type (MONO_TYPE_I1);
964 return type_to_simd_type (MONO_TYPE_R4);
967 return type_to_simd_type (MONO_TYPE_R8);
969 g_assert_not_reached ();
980 * Return the LLVM basic block corresponding to BB.
982 static LLVMBasicBlockRef
983 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
985 char bb_name_buf [128];
988 if (ctx->bblocks [bb->block_num].bblock == NULL) {
989 if (bb->flags & BB_EXCEPTION_HANDLER) {
990 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
991 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
992 bb_name = bb_name_buf;
993 } else if (bb->block_num < 256) {
994 if (!ctx->module->bb_names) {
995 ctx->module->bb_names_len = 256;
996 ctx->module->bb_names = g_new0 (char*, ctx->module->bb_names_len);
998 if (!ctx->module->bb_names [bb->block_num]) {
1001 n = g_strdup_printf ("BB%d", bb->block_num);
1002 mono_memory_barrier ();
1003 ctx->module->bb_names [bb->block_num] = n;
1005 bb_name = ctx->module->bb_names [bb->block_num];
1007 sprintf (bb_name_buf, "BB%d", bb->block_num);
1008 bb_name = bb_name_buf;
1011 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1012 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
1015 return ctx->bblocks [bb->block_num].bblock;
1021 * Return the last LLVM bblock corresponding to BB.
1022 * This might not be equal to the bb returned by get_bb () since we need to generate
1023 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
1025 static LLVMBasicBlockRef
1026 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
1029 return ctx->bblocks [bb->block_num].end_bblock;
1032 static LLVMBasicBlockRef
1033 gen_bb (EmitContext *ctx, const char *prefix)
1037 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
1038 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
1044 * Return the target of the patch identified by TYPE and TARGET.
1047 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1053 memset (&ji, 0, sizeof (ji));
1055 ji.data.target = target;
1057 res = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE, &error);
1058 mono_error_assert_ok (&error);
1066 * Emit code to convert the LLVM value V to DTYPE.
1069 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1071 LLVMTypeRef stype = LLVMTypeOf (v);
1073 if (stype != dtype) {
1074 gboolean ext = FALSE;
1077 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1079 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1081 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1085 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1087 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1088 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1091 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1092 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1093 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1094 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1095 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1096 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1097 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1098 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1100 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1101 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1102 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1103 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1104 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1105 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1107 if (mono_arch_is_soft_float ()) {
1108 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1109 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1110 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1111 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1114 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1115 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1118 LLVMDumpValue (LLVMConstNull (dtype));
1119 g_assert_not_reached ();
1127 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1129 return convert_full (ctx, v, dtype, FALSE);
1133 * emit_volatile_load:
1135 * If vreg is volatile, emit a load from its address.
1138 emit_volatile_load (EmitContext *ctx, int vreg)
1144 // FIXME: This hack is required because we pass the rgctx in a callee saved
1145 // register on arm64 (x15), and llvm might keep the value in that register
1146 // even through the register is marked as 'reserved' inside llvm.
1147 if (ctx->cfg->rgctx_var && ctx->cfg->rgctx_var->dreg == vreg)
1148 v = mono_llvm_build_load (ctx->builder, ctx->addresses [vreg], "", TRUE);
1150 v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1152 v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1154 t = ctx->vreg_cli_types [vreg];
1155 if (t && !t->byref) {
1157 * Might have to zero extend since llvm doesn't have
1160 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1161 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1162 else if (t->type == MONO_TYPE_I1 || t->type == MONO_TYPE_I2)
1163 v = LLVMBuildSExt (ctx->builder, v, LLVMInt32Type (), "");
1164 else if (t->type == MONO_TYPE_U8)
1165 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1172 * emit_volatile_store:
1174 * If VREG is volatile, emit a store from its value to its address.
1177 emit_volatile_store (EmitContext *ctx, int vreg)
1179 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1181 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1182 g_assert (ctx->addresses [vreg]);
1183 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1188 sig_to_llvm_sig_no_cinfo (EmitContext *ctx, MonoMethodSignature *sig)
1190 LLVMTypeRef ret_type;
1191 LLVMTypeRef *param_types = NULL;
1196 rtype = mini_get_underlying_type (sig->ret);
1197 ret_type = type_to_llvm_type (ctx, rtype);
1201 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1205 param_types [pindex ++] = ThisType ();
1206 for (i = 0; i < sig->param_count; ++i)
1207 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1209 if (!ctx_ok (ctx)) {
1210 g_free (param_types);
1214 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1215 g_free (param_types);
1221 * sig_to_llvm_sig_full:
1223 * Return the LLVM signature corresponding to the mono signature SIG using the
1224 * calling convention information in CINFO. Fill out the parameter mapping information in CINFO.
1227 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo)
1229 LLVMTypeRef ret_type;
1230 LLVMTypeRef *param_types = NULL;
1232 int i, j, pindex, vret_arg_pindex = 0;
1233 gboolean vretaddr = FALSE;
1237 return sig_to_llvm_sig_no_cinfo (ctx, sig);
1239 rtype = mini_get_underlying_type (sig->ret);
1240 ret_type = type_to_llvm_type (ctx, rtype);
1244 switch (cinfo->ret.storage) {
1245 case LLVMArgVtypeInReg:
1246 /* LLVM models this by returning an aggregate value */
1247 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1248 LLVMTypeRef members [2];
1250 members [0] = IntPtrType ();
1251 ret_type = LLVMStructType (members, 1, FALSE);
1252 } else if (cinfo->ret.pair_storage [0] == LLVMArgNone && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1254 ret_type = LLVMVoidType ();
1255 } else if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgInIReg) {
1256 LLVMTypeRef members [2];
1258 members [0] = IntPtrType ();
1259 members [1] = IntPtrType ();
1260 ret_type = LLVMStructType (members, 2, FALSE);
1262 g_assert_not_reached ();
1265 case LLVMArgVtypeByVal:
1266 /* Vtype returned normally by val */
1268 case LLVMArgVtypeAsScalar: {
1269 int size = mono_class_value_size (mono_class_from_mono_type (rtype), NULL);
1270 /* LLVM models this by returning an int */
1271 if (size < SIZEOF_VOID_P) {
1272 g_assert (cinfo->ret.nslots == 1);
1273 ret_type = LLVMIntType (size * 8);
1275 g_assert (cinfo->ret.nslots == 1 || cinfo->ret.nslots == 2);
1276 ret_type = LLVMIntType (cinfo->ret.nslots * sizeof (mgreg_t) * 8);
1280 case LLVMArgAsIArgs:
1281 ret_type = LLVMArrayType (IntPtrType (), cinfo->ret.nslots);
1283 case LLVMArgFpStruct: {
1284 /* Vtype returned as a fp struct */
1285 LLVMTypeRef members [16];
1287 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1288 for (i = 0; i < cinfo->ret.nslots; ++i)
1289 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1290 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1293 case LLVMArgVtypeByRef:
1294 /* Vtype returned using a hidden argument */
1295 ret_type = LLVMVoidType ();
1297 case LLVMArgVtypeRetAddr:
1298 case LLVMArgGsharedvtFixed:
1299 case LLVMArgGsharedvtFixedVtype:
1300 case LLVMArgGsharedvtVariable:
1302 ret_type = LLVMVoidType ();
1308 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1310 if (cinfo->ret.storage == LLVMArgVtypeByRef) {
1312 * Has to be the first argument because of the sret argument attribute
1313 * FIXME: This might conflict with passing 'this' as the first argument, but
1314 * this is only used on arm64 which has a dedicated struct return register.
1316 cinfo->vret_arg_pindex = pindex;
1317 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->ret);
1318 if (!ctx_ok (ctx)) {
1319 g_free (param_types);
1322 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1325 if (!ctx->llvm_only && cinfo->rgctx_arg) {
1326 cinfo->rgctx_arg_pindex = pindex;
1327 param_types [pindex] = ctx->module->ptr_type;
1330 if (cinfo->imt_arg) {
1331 cinfo->imt_arg_pindex = pindex;
1332 param_types [pindex] = ctx->module->ptr_type;
1336 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1337 vret_arg_pindex = pindex;
1338 if (cinfo->vret_arg_index == 1) {
1339 /* Add the slots consumed by the first argument */
1340 LLVMArgInfo *ainfo = &cinfo->args [0];
1341 switch (ainfo->storage) {
1342 case LLVMArgVtypeInReg:
1343 for (j = 0; j < 2; ++j) {
1344 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1353 cinfo->vret_arg_pindex = vret_arg_pindex;
1356 if (vretaddr && vret_arg_pindex == pindex)
1357 param_types [pindex ++] = IntPtrType ();
1359 cinfo->this_arg_pindex = pindex;
1360 param_types [pindex ++] = ThisType ();
1361 cinfo->args [0].pindex = cinfo->this_arg_pindex;
1363 if (vretaddr && vret_arg_pindex == pindex)
1364 param_types [pindex ++] = IntPtrType ();
1365 for (i = 0; i < sig->param_count; ++i) {
1366 LLVMArgInfo *ainfo = &cinfo->args [i + sig->hasthis];
1368 if (vretaddr && vret_arg_pindex == pindex)
1369 param_types [pindex ++] = IntPtrType ();
1370 ainfo->pindex = pindex;
1372 switch (ainfo->storage) {
1373 case LLVMArgVtypeInReg:
1374 for (j = 0; j < 2; ++j) {
1375 switch (ainfo->pair_storage [j]) {
1377 param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
1382 g_assert_not_reached ();
1386 case LLVMArgVtypeByVal:
1387 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1390 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1393 case LLVMArgAsIArgs:
1394 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1397 case LLVMArgVtypeByRef:
1398 param_types [pindex] = type_to_llvm_arg_type (ctx, ainfo->type);
1401 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1404 case LLVMArgAsFpArgs: {
1407 /* Emit dummy fp arguments if needed so the rest is passed on the stack */
1408 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
1409 param_types [pindex ++] = LLVMDoubleType ();
1410 for (j = 0; j < ainfo->nslots; ++j)
1411 param_types [pindex ++] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1414 case LLVMArgVtypeAsScalar:
1415 g_assert_not_reached ();
1417 case LLVMArgGsharedvtFixed:
1418 case LLVMArgGsharedvtFixedVtype:
1419 param_types [pindex ++] = LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0);
1421 case LLVMArgGsharedvtVariable:
1422 param_types [pindex ++] = LLVMPointerType (IntPtrType (), 0);
1425 param_types [pindex ++] = type_to_llvm_arg_type (ctx, ainfo->type);
1429 if (!ctx_ok (ctx)) {
1430 g_free (param_types);
1433 if (vretaddr && vret_arg_pindex == pindex)
1434 param_types [pindex ++] = IntPtrType ();
1435 if (ctx->llvm_only && cinfo->rgctx_arg) {
1436 /* Pass the rgctx as the last argument */
1437 cinfo->rgctx_arg_pindex = pindex;
1438 param_types [pindex] = ctx->module->ptr_type;
1442 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1443 g_free (param_types);
1449 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1451 return sig_to_llvm_sig_full (ctx, sig, NULL);
1455 * LLVMFunctionType1:
1457 * Create an LLVM function type from the arguments.
1459 static G_GNUC_UNUSED LLVMTypeRef
1460 LLVMFunctionType0 (LLVMTypeRef ReturnType,
1463 return LLVMFunctionType (ReturnType, NULL, 0, IsVarArg);
1467 * LLVMFunctionType1:
1469 * Create an LLVM function type from the arguments.
1471 static G_GNUC_UNUSED LLVMTypeRef
1472 LLVMFunctionType1 (LLVMTypeRef ReturnType,
1473 LLVMTypeRef ParamType1,
1476 LLVMTypeRef param_types [1];
1478 param_types [0] = ParamType1;
1480 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1484 * LLVMFunctionType2:
1486 * Create an LLVM function type from the arguments.
1488 static G_GNUC_UNUSED LLVMTypeRef
1489 LLVMFunctionType2 (LLVMTypeRef ReturnType,
1490 LLVMTypeRef ParamType1,
1491 LLVMTypeRef ParamType2,
1494 LLVMTypeRef param_types [2];
1496 param_types [0] = ParamType1;
1497 param_types [1] = ParamType2;
1499 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1503 * LLVMFunctionType3:
1505 * Create an LLVM function type from the arguments.
1507 static G_GNUC_UNUSED LLVMTypeRef
1508 LLVMFunctionType3 (LLVMTypeRef ReturnType,
1509 LLVMTypeRef ParamType1,
1510 LLVMTypeRef ParamType2,
1511 LLVMTypeRef ParamType3,
1514 LLVMTypeRef param_types [3];
1516 param_types [0] = ParamType1;
1517 param_types [1] = ParamType2;
1518 param_types [2] = ParamType3;
1520 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1523 static G_GNUC_UNUSED LLVMTypeRef
1524 LLVMFunctionType5 (LLVMTypeRef ReturnType,
1525 LLVMTypeRef ParamType1,
1526 LLVMTypeRef ParamType2,
1527 LLVMTypeRef ParamType3,
1528 LLVMTypeRef ParamType4,
1529 LLVMTypeRef ParamType5,
1532 LLVMTypeRef param_types [5];
1534 param_types [0] = ParamType1;
1535 param_types [1] = ParamType2;
1536 param_types [2] = ParamType3;
1537 param_types [3] = ParamType4;
1538 param_types [4] = ParamType5;
1540 return LLVMFunctionType (ReturnType, param_types, 5, IsVarArg);
1546 * Create an LLVM builder and remember it so it can be freed later.
1548 static LLVMBuilderRef
1549 create_builder (EmitContext *ctx)
1551 LLVMBuilderRef builder = LLVMCreateBuilder ();
1553 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1559 get_aotconst_name (MonoJumpInfoType type, gconstpointer data, int got_offset)
1564 case MONO_PATCH_INFO_INTERNAL_METHOD:
1565 name = g_strdup_printf ("jit_icall_%s", data);
1567 case MONO_PATCH_INFO_RGCTX_SLOT_INDEX: {
1568 MonoJumpInfoRgctxEntry *entry = (MonoJumpInfoRgctxEntry*)data;
1569 name = g_strdup_printf ("RGCTX_SLOT_INDEX_%s", mono_rgctx_info_type_to_str (entry->info_type));
1573 name = g_strdup_printf ("%s_%d", mono_ji_type_to_string (type), got_offset);
1581 get_aotconst_typed (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data, LLVMTypeRef llvm_type)
1585 LLVMValueRef indexes [2];
1586 LLVMValueRef got_entry_addr, load;
1587 LLVMBuilderRef builder = ctx->builder;
1592 MonoJumpInfo tmp_ji;
1594 tmp_ji.data.target = data;
1596 MonoJumpInfo *ji = mono_aot_patch_info_dup (&tmp_ji);
1598 ji->next = cfg->patch_info;
1599 cfg->patch_info = ji;
1601 got_offset = mono_aot_get_got_offset (cfg->patch_info);
1602 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
1604 * If the got slot is shared, it means its initialized when the aot image is loaded, so we don't need to
1605 * explicitly initialize it.
1607 if (!mono_aot_is_shared_got_offset (got_offset)) {
1608 //mono_print_ji (ji);
1610 ctx->has_got_access = TRUE;
1613 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1614 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
1615 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
1617 name = get_aotconst_name (type, data, got_offset);
1619 load = LLVMBuildLoad (builder, got_entry_addr, "");
1620 load = convert (ctx, load, llvm_type);
1621 LLVMSetValueName (load, name ? name : "");
1623 load = LLVMBuildLoad (builder, got_entry_addr, name ? name : "");
1626 //set_invariant_load_flag (load);
1632 get_aotconst (EmitContext *ctx, MonoJumpInfoType type, gconstpointer data)
1634 return get_aotconst_typed (ctx, type, data, NULL);
1638 get_callee (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1640 LLVMValueRef callee;
1642 if (ctx->llvm_only) {
1643 callee_name = mono_aot_get_direct_call_symbol (type, data);
1645 /* Directly callable */
1647 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->direct_callables, callee_name);
1649 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1651 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1653 g_hash_table_insert (ctx->module->direct_callables, (char*)callee_name, callee);
1655 /* LLVMTypeRef's are uniqued */
1656 if (LLVMGetElementType (LLVMTypeOf (callee)) != llvm_sig)
1657 return LLVMConstBitCast (callee, LLVMPointerType (llvm_sig, 0));
1659 g_free (callee_name);
1665 * Calls are made through the GOT.
1667 return get_aotconst_typed (ctx, type, data, LLVMPointerType (llvm_sig, 0));
1669 MonoJumpInfo *ji = NULL;
1671 callee_name = mono_aot_get_plt_symbol (type, data);
1675 if (ctx->cfg->compile_aot)
1676 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1677 mono_add_patch_info (ctx->cfg, 0, type, data);
1680 callee = (LLVMValueRef)g_hash_table_lookup (ctx->module->plt_entries, callee_name);
1682 callee = LLVMAddFunction (ctx->lmodule, callee_name, llvm_sig);
1684 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1686 g_hash_table_insert (ctx->module->plt_entries, (char*)callee_name, callee);
1689 if (ctx->cfg->compile_aot) {
1690 ji = g_new0 (MonoJumpInfo, 1);
1692 ji->data.target = data;
1694 g_hash_table_insert (ctx->module->plt_entries_ji, ji, callee);
1702 emit_jit_callee (EmitContext *ctx, const char *name, LLVMTypeRef llvm_sig, gpointer target)
1704 #if LLVM_API_VERSION > 100
1705 LLVMValueRef tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
1706 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
1707 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
1708 LLVMValueRef callee = LLVMBuildLoad (ctx->builder, tramp_var, "");
1711 LLVMValueRef callee = LLVMAddFunction (ctx->lmodule, "", llvm_sig);
1712 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
1718 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
1720 MonoMethodHeader *header = cfg->header;
1721 MonoExceptionClause *clause;
1725 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
1726 return (bb->region >> 8) - 1;
1729 for (i = 0; i < header->num_clauses; ++i) {
1730 clause = &header->clauses [i];
1732 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
1739 static MonoExceptionClause *
1740 get_most_deep_clause (MonoCompile *cfg, EmitContext *ctx, MonoBasicBlock *bb)
1742 // Since they're sorted by nesting we just need
1743 // the first one that the bb is a member of
1744 for (int i = 0; i < cfg->header->num_clauses; i++) {
1745 MonoExceptionClause *curr = &cfg->header->clauses [i];
1747 if (MONO_OFFSET_IN_CLAUSE (curr, bb->real_offset))
1755 set_metadata_flag (LLVMValueRef v, const char *flag_name)
1757 LLVMValueRef md_arg;
1760 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1761 md_arg = LLVMMDString ("mono", 4);
1762 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1766 set_invariant_load_flag (LLVMValueRef v)
1768 LLVMValueRef md_arg;
1770 const char *flag_name;
1772 // FIXME: Cache this
1773 flag_name = "invariant.load";
1774 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1775 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
1776 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1782 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
1786 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
1788 MonoCompile *cfg = ctx->cfg;
1789 LLVMValueRef lcall = NULL;
1790 LLVMBuilderRef builder = *builder_ref;
1791 MonoExceptionClause *clause;
1793 if (ctx->llvm_only) {
1794 clause = get_most_deep_clause (cfg, ctx, bb);
1797 g_assert (clause->flags == MONO_EXCEPTION_CLAUSE_NONE || clause->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1800 * Have to use an invoke instead of a call, branching to the
1801 * handler bblock of the clause containing this bblock.
1803 intptr_t key = CLAUSE_END(clause);
1805 LLVMBasicBlockRef lpad_bb = (LLVMBasicBlockRef)g_hash_table_lookup (ctx->exc_meta, (gconstpointer)key);
1807 // FIXME: Find the one that has the lowest end bound for the right start address
1808 // FIXME: Finally + nesting
1811 LLVMBasicBlockRef noex_bb = gen_bb (ctx, "CALL_NOEX_BB");
1814 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, lpad_bb, "");
1816 builder = ctx->builder = create_builder (ctx);
1817 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1819 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1823 int clause_index = get_handler_clause (cfg, bb);
1825 if (clause_index != -1) {
1826 MonoMethodHeader *header = cfg->header;
1827 MonoExceptionClause *ec = &header->clauses [clause_index];
1828 MonoBasicBlock *tblock;
1829 LLVMBasicBlockRef ex_bb, noex_bb;
1832 * Have to use an invoke instead of a call, branching to the
1833 * handler bblock of the clause containing this bblock.
1836 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1838 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
1841 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
1843 ex_bb = get_bb (ctx, tblock);
1845 noex_bb = gen_bb (ctx, "NOEX_BB");
1848 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1850 builder = ctx->builder = create_builder (ctx);
1851 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1853 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1858 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1859 ctx->builder = builder;
1863 *builder_ref = ctx->builder;
1869 emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, LLVMValueRef base, const char *name, gboolean is_faulting, BarrierKind barrier)
1871 const char *intrins_name;
1872 LLVMValueRef args [16], res;
1873 LLVMTypeRef addr_type;
1874 gboolean use_intrinsics = TRUE;
1876 #if LLVM_API_VERSION > 100
1877 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1878 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
1881 cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
1882 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
1883 *builder_ref = ctx->builder;
1884 use_intrinsics = FALSE;
1888 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
1889 LLVMAtomicOrdering ordering;
1892 case LLVM_BARRIER_NONE:
1893 ordering = LLVMAtomicOrderingNotAtomic;
1895 case LLVM_BARRIER_ACQ:
1896 ordering = LLVMAtomicOrderingAcquire;
1898 case LLVM_BARRIER_SEQ:
1899 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1902 g_assert_not_reached ();
1907 * We handle loads which can fault by calling a mono specific intrinsic
1908 * using an invoke, so they are handled properly inside try blocks.
1909 * We can't use this outside clauses, since LLVM optimizes intrinsics which
1910 * are marked with IntrReadArgMem.
1914 intrins_name = "llvm.mono.load.i8.p0i8";
1917 intrins_name = "llvm.mono.load.i16.p0i16";
1920 intrins_name = "llvm.mono.load.i32.p0i32";
1923 intrins_name = "llvm.mono.load.i64.p0i64";
1926 g_assert_not_reached ();
1929 addr_type = LLVMTypeOf (addr);
1930 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
1931 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1934 args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1935 args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1936 args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1937 res = emit_call (ctx, bb, builder_ref, get_intrinsic (ctx, intrins_name), args, 4);
1939 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
1940 res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
1941 else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
1942 res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
1949 * We emit volatile loads for loads which can fault, because otherwise
1950 * LLVM will generate invalid code when encountering a load from a
1953 if (barrier != LLVM_BARRIER_NONE)
1954 res = mono_llvm_build_atomic_load (*builder_ref, addr, name, is_faulting, size, barrier);
1956 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting);
1958 /* Mark it with a custom metadata */
1961 set_metadata_flag (res, "mono.faulting.load");
1969 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
1971 return emit_load_general (ctx, bb, builder_ref, size, addr, addr, name, is_faulting, LLVM_BARRIER_NONE);
1975 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting, BarrierKind barrier)
1977 const char *intrins_name;
1978 LLVMValueRef args [16];
1979 gboolean use_intrinsics = TRUE;
1981 #if LLVM_API_VERSION > 100
1982 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only) {
1983 /* The llvm.mono.load/store intrinsics are not supported by this llvm version, emit an explicit null check instead */
1984 LLVMValueRef cmp = LLVMBuildICmp (*builder_ref, LLVMIntEQ, base, LLVMConstNull (LLVMTypeOf (base)), "");
1985 emit_cond_system_exception (ctx, bb, "NullReferenceException", cmp);
1986 *builder_ref = ctx->builder;
1987 use_intrinsics = FALSE;
1991 if (is_faulting && bb->region != -1 && !ctx->cfg->llvm_only && use_intrinsics) {
1992 LLVMAtomicOrdering ordering;
1995 case LLVM_BARRIER_NONE:
1996 ordering = LLVMAtomicOrderingNotAtomic;
1998 case LLVM_BARRIER_REL:
1999 ordering = LLVMAtomicOrderingRelease;
2001 case LLVM_BARRIER_SEQ:
2002 ordering = LLVMAtomicOrderingSequentiallyConsistent;
2005 g_assert_not_reached ();
2011 intrins_name = "llvm.mono.store.i8.p0i8";
2014 intrins_name = "llvm.mono.store.i16.p0i16";
2017 intrins_name = "llvm.mono.store.i32.p0i32";
2020 intrins_name = "llvm.mono.store.i64.p0i64";
2023 g_assert_not_reached ();
2026 if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
2027 value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
2028 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
2033 args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2034 args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
2035 args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
2036 emit_call (ctx, bb, builder_ref, get_intrinsic (ctx, intrins_name), args, 5);
2038 if (barrier != LLVM_BARRIER_NONE)
2039 mono_llvm_build_aligned_store (*builder_ref, value, addr, barrier, size);
2041 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
2046 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, LLVMValueRef base, gboolean is_faulting)
2048 emit_store_general (ctx, bb, builder_ref, size, value, addr, base, is_faulting, LLVM_BARRIER_NONE);
2052 * emit_cond_system_exception:
2054 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
2055 * Might set the ctx exception.
2058 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
2060 LLVMBasicBlockRef ex_bb, ex2_bb = NULL, noex_bb;
2061 LLVMBuilderRef builder;
2062 MonoClass *exc_class;
2063 LLVMValueRef args [2];
2064 LLVMValueRef callee;
2065 gboolean no_pc = FALSE;
2067 if (IS_TARGET_AMD64)
2068 /* Some platforms don't require the pc argument */
2071 ex_bb = gen_bb (ctx, "EX_BB");
2073 ex2_bb = gen_bb (ctx, "EX2_BB");
2074 noex_bb = gen_bb (ctx, "NOEX_BB");
2076 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
2078 exc_class = mono_class_load_from_name (mono_get_corlib (), "System", exc_type);
2080 /* Emit exception throwing code */
2081 ctx->builder = builder = create_builder (ctx);
2082 LLVMPositionBuilderAtEnd (builder, ex_bb);
2084 if (ctx->cfg->llvm_only) {
2085 static LLVMTypeRef sig;
2088 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2089 callee = get_callee (ctx, sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, "mono_llvm_throw_corlib_exception");
2091 LLVMBuildBr (builder, ex2_bb);
2093 ctx->builder = builder = create_builder (ctx);
2094 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2096 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2097 emit_call (ctx, bb, &builder, callee, args, 1);
2098 LLVMBuildUnreachable (builder);
2100 ctx->builder = builder = create_builder (ctx);
2101 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2103 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2109 callee = ctx->module->throw_corlib_exception;
2112 const char *icall_name;
2115 sig = LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE);
2117 sig = LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), LLVMPointerType (LLVMInt8Type (), 0), FALSE);
2118 icall_name = "llvm_throw_corlib_exception_abs_trampoline";
2120 if (ctx->cfg->compile_aot) {
2121 callee = get_callee (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
2124 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
2125 * - On x86, LLVM generated code doesn't push the arguments
2126 * - The trampoline takes the throw address as an arguments, not a pc offset.
2128 gpointer target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
2129 callee = emit_jit_callee (ctx, "llvm_throw_corlib_exception_trampoline", sig, target);
2131 #if LLVM_API_VERSION > 100
2133 * Make sure that ex_bb starts with the invoke, so the block address points to it, and not to the load
2134 * added by emit_jit_callee ().
2136 ex2_bb = gen_bb (ctx, "EX2_BB");
2137 LLVMBuildBr (builder, ex2_bb);
2140 ctx->builder = builder = create_builder (ctx);
2141 LLVMPositionBuilderAtEnd (ctx->builder, ex2_bb);
2143 mono_memory_barrier ();
2144 ctx->module->throw_corlib_exception = callee;
2149 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
2152 * The LLVM mono branch contains changes so a block address can be passed as an
2153 * argument to a call.
2156 emit_call (ctx, bb, &builder, callee, args, 1);
2158 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
2159 emit_call (ctx, bb, &builder, callee, args, 2);
2162 LLVMBuildUnreachable (builder);
2164 ctx->builder = builder = create_builder (ctx);
2165 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
2167 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
2174 * emit_args_to_vtype:
2176 * Emit code to store the vtype in the arguments args to the address ADDRESS.
2179 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
2181 int j, size, nslots;
2183 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
2185 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2186 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2189 if (ainfo->storage == LLVMArgAsFpArgs)
2190 nslots = ainfo->nslots;
2194 for (j = 0; j < nslots; ++j) {
2195 LLVMValueRef index [2], addr, daddr;
2196 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2197 LLVMTypeRef part_type;
2199 while (part_size != 1 && part_size != 2 && part_size != 4 && part_size < 8)
2202 if (ainfo->pair_storage [j] == LLVMArgNone)
2205 switch (ainfo->pair_storage [j]) {
2206 case LLVMArgInIReg: {
2207 part_type = LLVMIntType (part_size * 8);
2208 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2209 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2210 addr = LLVMBuildGEP (builder, address, index, 1, "");
2212 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2213 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2214 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2216 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
2219 case LLVMArgInFPReg: {
2220 LLVMTypeRef arg_type;
2222 if (ainfo->esize == 8)
2223 arg_type = LLVMDoubleType ();
2225 arg_type = LLVMFloatType ();
2227 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2228 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2229 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2230 LLVMBuildStore (builder, args [j], addr);
2236 g_assert_not_reached ();
2239 size -= sizeof (gpointer);
2244 * emit_vtype_to_args:
2246 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
2247 * into ARGS, and the number of arguments into NARGS.
2250 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
2253 int j, size, nslots;
2254 LLVMTypeRef arg_type;
2256 size = get_vtype_size (t);
2258 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t)))
2259 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
2261 if (ainfo->storage == LLVMArgAsFpArgs)
2262 nslots = ainfo->nslots;
2265 for (j = 0; j < nslots; ++j) {
2266 LLVMValueRef index [2], addr, daddr;
2267 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
2269 if (ainfo->pair_storage [j] == LLVMArgNone)
2272 switch (ainfo->pair_storage [j]) {
2274 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
2275 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
2276 addr = LLVMBuildGEP (builder, address, index, 1, "");
2278 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
2279 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2280 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2282 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
2284 case LLVMArgInFPReg:
2285 if (ainfo->esize == 8)
2286 arg_type = LLVMDoubleType ();
2288 arg_type = LLVMFloatType ();
2289 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
2290 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
2291 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
2292 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
2297 g_assert_not_reached ();
2299 size -= sizeof (gpointer);
2306 build_alloca_llvm_type_name (EmitContext *ctx, LLVMTypeRef t, int align, const char *name)
2309 * Have to place all alloca's at the end of the entry bb, since otherwise they would
2310 * get executed every time control reaches them.
2312 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
2314 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, name);
2315 return ctx->last_alloca;
2319 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
2321 return build_alloca_llvm_type_name (ctx, t, align, "");
2325 build_alloca (EmitContext *ctx, MonoType *t)
2327 MonoClass *k = mono_class_from_mono_type (t);
2330 g_assert (!mini_is_gsharedvt_variable_type (t));
2332 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
2335 align = mono_class_min_align (k);
2337 /* Sometimes align is not a power of 2 */
2338 while (mono_is_power_of_two (align) == -1)
2341 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
2345 emit_gsharedvt_ldaddr (EmitContext *ctx, int vreg)
2349 * Compute the address of the local as gsharedvt_locals_var + gsharedvt_info_var->locals_offsets [idx].
2351 MonoCompile *cfg = ctx->cfg;
2352 LLVMBuilderRef builder = ctx->builder;
2353 LLVMValueRef offset, offset_var;
2354 LLVMValueRef info_var = ctx->values [cfg->gsharedvt_info_var->dreg];
2355 LLVMValueRef locals_var = ctx->values [cfg->gsharedvt_locals_var->dreg];
2359 g_assert (info_var);
2360 g_assert (locals_var);
2362 int idx = cfg->gsharedvt_vreg_to_idx [vreg] - 1;
2364 offset = LLVMConstInt (LLVMInt32Type (), MONO_STRUCT_OFFSET (MonoGSharedVtMethodRuntimeInfo, entries) + (idx * sizeof (gpointer)), FALSE);
2365 ptr = LLVMBuildAdd (builder, convert (ctx, info_var, IntPtrType ()), convert (ctx, offset, IntPtrType ()), "");
2367 name = g_strdup_printf ("gsharedvt_local_%d_offset", vreg);
2368 offset_var = LLVMBuildLoad (builder, convert (ctx, ptr, LLVMPointerType (LLVMInt32Type (), 0)), name);
2370 return LLVMBuildAdd (builder, convert (ctx, locals_var, IntPtrType ()), convert (ctx, offset_var, IntPtrType ()), "");
2374 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
2377 mark_as_used (MonoLLVMModule *module, LLVMValueRef global)
2380 module->used = g_ptr_array_sized_new (16);
2381 g_ptr_array_add (module->used, global);
2385 emit_llvm_used (MonoLLVMModule *module)
2387 LLVMModuleRef lmodule = module->lmodule;
2388 LLVMTypeRef used_type;
2389 LLVMValueRef used, *used_elem;
2395 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), module->used->len);
2396 used = LLVMAddGlobal (lmodule, used_type, "llvm.used");
2397 used_elem = g_new0 (LLVMValueRef, module->used->len);
2398 for (i = 0; i < module->used->len; ++i)
2399 used_elem [i] = LLVMConstBitCast ((LLVMValueRef)g_ptr_array_index (module->used, i), LLVMPointerType (LLVMInt8Type (), 0));
2400 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, module->used->len));
2401 LLVMSetLinkage (used, LLVMAppendingLinkage);
2402 LLVMSetSection (used, "llvm.metadata");
2408 * Emit a function mapping method indexes to their code
2411 emit_get_method (MonoLLVMModule *module)
2413 LLVMModuleRef lmodule = module->lmodule;
2414 LLVMValueRef func, switch_ins, m;
2415 LLVMBasicBlockRef entry_bb, fail_bb, bb, code_start_bb, code_end_bb;
2416 LLVMBasicBlockRef *bbs;
2418 LLVMBuilderRef builder = LLVMCreateBuilder ();
2423 * Emit a switch statement. Emitting a table of function addresses is smaller/faster,
2424 * but generating code seems safer.
2426 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2427 func = LLVMAddFunction (lmodule, module->get_method_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2428 LLVMSetLinkage (func, LLVMExternalLinkage);
2429 LLVMSetVisibility (func, LLVMHiddenVisibility);
2430 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2431 module->get_method = func;
2433 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2436 * Return llvm_code_start/llvm_code_end when called with -1/-2.
2437 * Hopefully, the toolchain doesn't reorder these functions. If it does,
2438 * then we will have to find another solution.
2441 name = g_strdup_printf ("BB_CODE_START");
2442 code_start_bb = LLVMAppendBasicBlock (func, name);
2444 LLVMPositionBuilderAtEnd (builder, code_start_bb);
2445 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_start, rtype, ""));
2447 name = g_strdup_printf ("BB_CODE_END");
2448 code_end_bb = LLVMAppendBasicBlock (func, name);
2450 LLVMPositionBuilderAtEnd (builder, code_end_bb);
2451 LLVMBuildRet (builder, LLVMBuildBitCast (builder, module->code_end, rtype, ""));
2453 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2454 for (i = 0; i < module->max_method_idx + 1; ++i) {
2455 name = g_strdup_printf ("BB_%d", i);
2456 bb = LLVMAppendBasicBlock (func, name);
2460 LLVMPositionBuilderAtEnd (builder, bb);
2462 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_lmethod, GINT_TO_POINTER (i));
2464 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2466 LLVMBuildRet (builder, LLVMConstNull (rtype));
2469 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2470 LLVMPositionBuilderAtEnd (builder, fail_bb);
2471 LLVMBuildRet (builder, LLVMConstNull (rtype));
2473 LLVMPositionBuilderAtEnd (builder, entry_bb);
2475 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2476 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -1, FALSE), code_start_bb);
2477 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), -2, FALSE), code_end_bb);
2478 for (i = 0; i < module->max_method_idx + 1; ++i) {
2479 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2482 mark_as_used (module, func);
2484 LLVMDisposeBuilder (builder);
2488 * emit_get_unbox_tramp:
2490 * Emit a function mapping method indexes to their unbox trampoline
2493 emit_get_unbox_tramp (MonoLLVMModule *module)
2495 LLVMModuleRef lmodule = module->lmodule;
2496 LLVMValueRef func, switch_ins, m;
2497 LLVMBasicBlockRef entry_bb, fail_bb, bb;
2498 LLVMBasicBlockRef *bbs;
2500 LLVMBuilderRef builder = LLVMCreateBuilder ();
2504 /* Similar to emit_get_method () */
2506 rtype = LLVMPointerType (LLVMInt8Type (), 0);
2507 func = LLVMAddFunction (lmodule, module->get_unbox_tramp_symbol, LLVMFunctionType1 (rtype, LLVMInt32Type (), FALSE));
2508 LLVMSetLinkage (func, LLVMExternalLinkage);
2509 LLVMSetVisibility (func, LLVMHiddenVisibility);
2510 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2511 module->get_unbox_tramp = func;
2513 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2515 bbs = g_new0 (LLVMBasicBlockRef, module->max_method_idx + 1);
2516 for (i = 0; i < module->max_method_idx + 1; ++i) {
2517 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2521 name = g_strdup_printf ("BB_%d", i);
2522 bb = LLVMAppendBasicBlock (func, name);
2526 LLVMPositionBuilderAtEnd (builder, bb);
2528 LLVMBuildRet (builder, LLVMBuildBitCast (builder, m, rtype, ""));
2531 fail_bb = LLVMAppendBasicBlock (func, "FAIL");
2532 LLVMPositionBuilderAtEnd (builder, fail_bb);
2533 LLVMBuildRet (builder, LLVMConstNull (rtype));
2535 LLVMPositionBuilderAtEnd (builder, entry_bb);
2537 switch_ins = LLVMBuildSwitch (builder, LLVMGetParam (func, 0), fail_bb, 0);
2538 for (i = 0; i < module->max_method_idx + 1; ++i) {
2539 m = (LLVMValueRef)g_hash_table_lookup (module->idx_to_unbox_tramp, GINT_TO_POINTER (i));
2543 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
2546 mark_as_used (module, func);
2547 LLVMDisposeBuilder (builder);
2550 /* Add a function to mark the beginning of LLVM code */
2552 emit_llvm_code_start (MonoLLVMModule *module)
2554 LLVMModuleRef lmodule = module->lmodule;
2556 LLVMBasicBlockRef entry_bb;
2557 LLVMBuilderRef builder;
2559 func = LLVMAddFunction (lmodule, "llvm_code_start", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2560 LLVMSetLinkage (func, LLVMInternalLinkage);
2561 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2562 module->code_start = func;
2563 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2564 builder = LLVMCreateBuilder ();
2565 LLVMPositionBuilderAtEnd (builder, entry_bb);
2566 LLVMBuildRetVoid (builder);
2567 LLVMDisposeBuilder (builder);
2571 emit_init_icall_wrapper (MonoLLVMModule *module, const char *name, const char *icall_name, int subtype)
2573 LLVMModuleRef lmodule = module->lmodule;
2574 LLVMValueRef func, indexes [2], got_entry_addr, args [16], callee;
2575 LLVMBasicBlockRef entry_bb;
2576 LLVMBuilderRef builder;
2583 func = LLVMAddFunction (lmodule, name, LLVMFunctionType1 (LLVMVoidType (), LLVMInt32Type (), FALSE));
2584 sig = LLVMFunctionType2 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), FALSE);
2589 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), IntPtrType (), FALSE));
2590 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), IntPtrType (), FALSE);
2593 func = LLVMAddFunction (lmodule, name, LLVMFunctionType2 (LLVMVoidType (), LLVMInt32Type (), ObjRefType (), FALSE));
2594 sig = LLVMFunctionType3 (LLVMVoidType (), IntPtrType (), LLVMInt32Type (), ObjRefType (), FALSE);
2597 g_assert_not_reached ();
2599 LLVMSetLinkage (func, LLVMInternalLinkage);
2600 LLVMAddFunctionAttr (func, LLVMNoInlineAttribute);
2601 mono_llvm_set_preserveall_cc (func);
2602 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2603 builder = LLVMCreateBuilder ();
2604 LLVMPositionBuilderAtEnd (builder, entry_bb);
2607 ji = g_new0 (MonoJumpInfo, 1);
2608 ji->type = MONO_PATCH_INFO_AOT_MODULE;
2609 ji = mono_aot_patch_info_dup (ji);
2610 got_offset = mono_aot_get_got_offset (ji);
2611 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2612 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2613 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2614 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2615 args [0] = LLVMBuildPtrToInt (builder, LLVMBuildLoad (builder, got_entry_addr, ""), IntPtrType (), "");
2616 args [1] = LLVMGetParam (func, 0);
2618 args [2] = LLVMGetParam (func, 1);
2620 ji = g_new0 (MonoJumpInfo, 1);
2621 ji->type = MONO_PATCH_INFO_INTERNAL_METHOD;
2622 ji->data.name = icall_name;
2623 ji = mono_aot_patch_info_dup (ji);
2624 got_offset = mono_aot_get_got_offset (ji);
2625 module->max_got_offset = MAX (module->max_got_offset, got_offset);
2626 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2627 indexes [1] = LLVMConstInt (LLVMInt32Type (), got_offset, FALSE);
2628 got_entry_addr = LLVMBuildGEP (builder, module->got_var, indexes, 2, "");
2629 callee = LLVMBuildLoad (builder, got_entry_addr, "");
2630 callee = LLVMBuildBitCast (builder, callee, LLVMPointerType (sig, 0), "");
2631 LLVMBuildCall (builder, callee, args, LLVMCountParamTypes (sig), "");
2633 // Set the inited flag
2634 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2635 indexes [1] = LLVMGetParam (func, 0);
2636 LLVMBuildStore (builder, LLVMConstInt (LLVMInt8Type (), 1, FALSE), LLVMBuildGEP (builder, module->inited_var, indexes, 2, ""));
2638 LLVMBuildRetVoid (builder);
2640 LLVMVerifyFunction(func, LLVMAbortProcessAction);
2641 LLVMDisposeBuilder (builder);
2646 * Emit wrappers around the C icalls used to initialize llvm methods, to
2647 * make the calling code smaller and to enable usage of the llvm
2648 * PreserveAll calling convention.
2651 emit_init_icall_wrappers (MonoLLVMModule *module)
2653 module->init_method = emit_init_icall_wrapper (module, "init_method", "mono_aot_init_llvm_method", 0);
2654 module->init_method_gshared_mrgctx = emit_init_icall_wrapper (module, "init_method_gshared_mrgctx", "mono_aot_init_gshared_method_mrgctx", 1);
2655 module->init_method_gshared_this = emit_init_icall_wrapper (module, "init_method_gshared_this", "mono_aot_init_gshared_method_this", 2);
2656 module->init_method_gshared_vtable = emit_init_icall_wrapper (module, "init_method_gshared_vtable", "mono_aot_init_gshared_method_vtable", 3);
2660 emit_llvm_code_end (MonoLLVMModule *module)
2662 LLVMModuleRef lmodule = module->lmodule;
2664 LLVMBasicBlockRef entry_bb;
2665 LLVMBuilderRef builder;
2667 func = LLVMAddFunction (lmodule, "llvm_code_end", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
2668 LLVMSetLinkage (func, LLVMInternalLinkage);
2669 LLVMAddFunctionAttr (func, LLVMNoUnwindAttribute);
2670 module->code_end = func;
2671 entry_bb = LLVMAppendBasicBlock (func, "ENTRY");
2672 builder = LLVMCreateBuilder ();
2673 LLVMPositionBuilderAtEnd (builder, entry_bb);
2674 LLVMBuildRetVoid (builder);
2675 LLVMDisposeBuilder (builder);
2679 emit_div_check (EmitContext *ctx, LLVMBuilderRef builder, MonoBasicBlock *bb, MonoInst *ins, LLVMValueRef lhs, LLVMValueRef rhs)
2681 gboolean need_div_check = ctx->cfg->backend->need_div_check;
2684 /* LLVM doesn't know that these can throw an exception since they are not called through an intrinsic */
2685 need_div_check = TRUE;
2687 if (!need_div_check)
2690 switch (ins->opcode) {
2703 case OP_IDIV_UN_IMM:
2704 case OP_LDIV_UN_IMM:
2705 case OP_IREM_UN_IMM:
2706 case OP_LREM_UN_IMM: {
2708 gboolean is_signed = (ins->opcode == OP_IDIV || ins->opcode == OP_LDIV || ins->opcode == OP_IREM || ins->opcode == OP_LREM ||
2709 ins->opcode == OP_IDIV_IMM || ins->opcode == OP_LDIV_IMM || ins->opcode == OP_IREM_IMM || ins->opcode == OP_LREM_IMM);
2711 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
2712 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
2715 builder = ctx->builder;
2717 /* b == -1 && a == 0x80000000 */
2719 LLVMValueRef c = (LLVMTypeOf (lhs) == LLVMInt32Type ()) ? LLVMConstInt (LLVMTypeOf (lhs), 0x80000000, FALSE) : LLVMConstInt (LLVMTypeOf (lhs), 0x8000000000000000LL, FALSE);
2720 LLVMValueRef cond1 = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), -1, FALSE), "");
2721 LLVMValueRef cond2 = LLVMBuildICmp (builder, LLVMIntEQ, lhs, c, "");
2723 cmp = LLVMBuildICmp (builder, LLVMIntEQ, LLVMBuildAnd (builder, cond1, cond2, ""), LLVMConstInt (LLVMInt1Type (), 1, FALSE), "");
2724 emit_cond_system_exception (ctx, bb, "OverflowException", cmp);
2727 builder = ctx->builder;
2739 * Emit code to initialize the GOT slots used by the method.
2742 emit_init_method (EmitContext *ctx)
2744 LLVMValueRef indexes [16], args [16], callee;
2745 LLVMValueRef inited_var, cmp, call;
2746 LLVMBasicBlockRef inited_bb, notinited_bb;
2747 LLVMBuilderRef builder = ctx->builder;
2748 MonoCompile *cfg = ctx->cfg;
2750 ctx->module->max_inited_idx = MAX (ctx->module->max_inited_idx, cfg->method_index);
2752 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2753 indexes [1] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, FALSE);
2754 inited_var = LLVMBuildLoad (builder, LLVMBuildGEP (builder, ctx->module->inited_var, indexes, 2, ""), "is_inited");
2756 args [0] = inited_var;
2757 args [1] = LLVMConstInt (LLVMInt8Type (), 1, FALSE);
2758 inited_var = LLVMBuildCall (ctx->builder, get_intrinsic (ctx, "llvm.expect.i8"), args, 2, "");
2760 cmp = LLVMBuildICmp (builder, LLVMIntEQ, inited_var, LLVMConstInt (LLVMTypeOf (inited_var), 0, FALSE), "");
2762 inited_bb = ctx->inited_bb;
2763 notinited_bb = gen_bb (ctx, "NOTINITED_BB");
2765 LLVMBuildCondBr (ctx->builder, cmp, notinited_bb, inited_bb);
2767 builder = ctx->builder = create_builder (ctx);
2768 LLVMPositionBuilderAtEnd (ctx->builder, notinited_bb);
2771 if (ctx->rgctx_arg && cfg->method->is_inflated && mono_method_get_context (cfg->method)->method_inst) {
2772 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2773 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2774 callee = ctx->module->init_method_gshared_mrgctx;
2775 call = LLVMBuildCall (builder, callee, args, 2, "");
2776 } else if (ctx->rgctx_arg) {
2777 /* A vtable is passed as the rgctx argument */
2778 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2779 args [1] = convert (ctx, ctx->rgctx_arg, IntPtrType ());
2780 callee = ctx->module->init_method_gshared_vtable;
2781 call = LLVMBuildCall (builder, callee, args, 2, "");
2782 } else if (cfg->gshared) {
2783 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2784 args [1] = convert (ctx, ctx->this_arg, ObjRefType ());
2785 callee = ctx->module->init_method_gshared_this;
2786 call = LLVMBuildCall (builder, callee, args, 2, "");
2788 args [0] = LLVMConstInt (LLVMInt32Type (), cfg->method_index, 0);
2789 callee = ctx->module->init_method;
2790 call = LLVMBuildCall (builder, callee, args, 1, "");
2794 * This enables llvm to keep arguments in their original registers/
2795 * scratch registers, since the call will not clobber them.
2797 mono_llvm_set_call_preserveall_cc (call);
2799 LLVMBuildBr (builder, inited_bb);
2800 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = inited_bb;
2802 builder = ctx->builder = create_builder (ctx);
2803 LLVMPositionBuilderAtEnd (ctx->builder, inited_bb);
2807 emit_unbox_tramp (EmitContext *ctx, const char *method_name, LLVMTypeRef method_type, LLVMValueRef method, int method_index)
2810 * Emit unbox trampoline using a tail call
2812 LLVMValueRef tramp, call, *args;
2813 LLVMBuilderRef builder;
2814 LLVMBasicBlockRef lbb;
2815 LLVMCallInfo *linfo;
2819 tramp_name = g_strdup_printf ("ut_%s", method_name);
2820 tramp = LLVMAddFunction (ctx->module->lmodule, tramp_name, method_type);
2821 LLVMSetLinkage (tramp, LLVMInternalLinkage);
2822 LLVMAddFunctionAttr (tramp, LLVMOptimizeForSizeAttribute);
2823 //LLVMAddFunctionAttr (tramp, LLVMNoUnwindAttribute);
2825 // FIXME: Reduce code duplication with mono_llvm_compile_method () etc.
2826 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2827 LLVMAddAttribute (LLVMGetParam (tramp, ctx->rgctx_arg_pindex), LLVMInRegAttribute);
2828 if (ctx->cfg->vret_addr) {
2829 LLVMSetValueName (LLVMGetParam (tramp, linfo->vret_arg_pindex), "vret");
2830 if (linfo->ret.storage == LLVMArgVtypeByRef) {
2831 LLVMAddAttribute (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVMStructRetAttribute);
2832 LLVMAddAttribute (LLVMGetParam (tramp, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
2836 lbb = LLVMAppendBasicBlock (tramp, "");
2837 builder = LLVMCreateBuilder ();
2838 LLVMPositionBuilderAtEnd (builder, lbb);
2840 nargs = LLVMCountParamTypes (method_type);
2841 args = g_new0 (LLVMValueRef, nargs);
2842 for (i = 0; i < nargs; ++i) {
2843 args [i] = LLVMGetParam (tramp, i);
2844 if (i == ctx->this_arg_pindex) {
2845 LLVMTypeRef arg_type = LLVMTypeOf (args [i]);
2847 args [i] = LLVMBuildPtrToInt (builder, args [i], IntPtrType (), "");
2848 args [i] = LLVMBuildAdd (builder, args [i], LLVMConstInt (IntPtrType (), sizeof (MonoObject), FALSE), "");
2849 args [i] = LLVMBuildIntToPtr (builder, args [i], arg_type, "");
2852 call = LLVMBuildCall (builder, method, args, nargs, "");
2853 if (!ctx->llvm_only && ctx->rgctx_arg_pindex != -1)
2854 LLVMAddInstrAttribute (call, 1 + ctx->rgctx_arg_pindex, LLVMInRegAttribute);
2855 if (linfo->ret.storage == LLVMArgVtypeByRef)
2856 LLVMAddInstrAttribute (call, 1 + linfo->vret_arg_pindex, LLVMStructRetAttribute);
2858 // FIXME: This causes assertions in clang
2859 //mono_llvm_set_must_tail (call);
2860 if (LLVMGetReturnType (method_type) == LLVMVoidType ())
2861 LLVMBuildRetVoid (builder);
2863 LLVMBuildRet (builder, call);
2865 g_hash_table_insert (ctx->module->idx_to_unbox_tramp, GINT_TO_POINTER (method_index), tramp);
2866 LLVMDisposeBuilder (builder);
2872 * Emit code to load/convert arguments.
2875 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
2878 MonoCompile *cfg = ctx->cfg;
2879 MonoMethodSignature *sig = ctx->sig;
2880 LLVMCallInfo *linfo = ctx->linfo;
2884 LLVMBuilderRef old_builder = ctx->builder;
2885 ctx->builder = builder;
2887 ctx->alloca_builder = create_builder (ctx);
2890 * Handle indirect/volatile variables by allocating memory for them
2891 * using 'alloca', and storing their address in a temporary.
2893 for (i = 0; i < cfg->num_varinfo; ++i) {
2894 MonoInst *var = cfg->varinfo [i];
2897 if (var->opcode == OP_GSHAREDVT_LOCAL || var->opcode == OP_GSHAREDVT_ARG_REGOFFSET) {
2898 } else if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || (mini_type_is_vtype (var->inst_vtype) && !MONO_CLASS_IS_SIMD (ctx->cfg, var->klass))) {
2899 vtype = type_to_llvm_type (ctx, var->inst_vtype);
2902 /* Could be already created by an OP_VPHI */
2903 if (!ctx->addresses [var->dreg]) {
2904 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
2905 //LLVMSetValueName (ctx->addresses [var->dreg], g_strdup_printf ("vreg_loc_%d", var->dreg));
2907 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
2911 names = g_new (char *, sig->param_count);
2912 mono_method_get_param_names (cfg->method, (const char **) names);
2914 for (i = 0; i < sig->param_count; ++i) {
2915 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
2916 int reg = cfg->args [i + sig->hasthis]->dreg;
2919 pindex = ainfo->pindex;
2921 switch (ainfo->storage) {
2922 case LLVMArgVtypeInReg:
2923 case LLVMArgAsFpArgs: {
2924 LLVMValueRef args [8];
2927 pindex += ainfo->ndummy_fpargs;
2929 /* The argument is received as a set of int/fp arguments, store them into the real argument */
2930 memset (args, 0, sizeof (args));
2931 if (ainfo->storage == LLVMArgVtypeInReg) {
2932 args [0] = LLVMGetParam (ctx->lmethod, pindex);
2933 if (ainfo->pair_storage [1] != LLVMArgNone)
2934 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
2936 g_assert (ainfo->nslots <= 8);
2937 for (j = 0; j < ainfo->nslots; ++j)
2938 args [j] = LLVMGetParam (ctx->lmethod, pindex + j);
2940 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2942 emit_args_to_vtype (ctx, builder, ainfo->type, ctx->addresses [reg], ainfo, args);
2944 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2945 /* Treat these as normal values */
2946 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2950 case LLVMArgVtypeByVal: {
2951 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2953 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (ainfo->type))) {
2954 /* Treat these as normal values */
2955 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2959 case LLVMArgVtypeByRef: {
2960 /* The argument is passed by ref */
2961 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
2964 case LLVMArgAsIArgs: {
2965 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2968 /* The argument is received as an array of ints, store it into the real argument */
2969 ctx->addresses [reg] = build_alloca (ctx, ainfo->type);
2971 size = mono_class_value_size (mono_class_from_mono_type (ainfo->type), NULL);
2972 if (size < SIZEOF_VOID_P) {
2973 /* The upper bits of the registers might not be valid */
2974 LLVMValueRef val = LLVMBuildExtractValue (builder, arg, 0, "");
2975 LLVMValueRef dest = convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMIntType (size * 8), 0));
2976 LLVMBuildStore (ctx->builder, LLVMBuildTrunc (builder, val, LLVMIntType (size * 8), ""), dest);
2978 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
2982 case LLVMArgVtypeAsScalar:
2983 g_assert_not_reached ();
2985 case LLVMArgGsharedvtFixed: {
2986 /* These are non-gsharedvt arguments passed by ref, the rest of the IR treats them as scalars */
2987 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
2990 name = g_strdup_printf ("arg_%s", names [i]);
2992 name = g_strdup_printf ("arg_%d", i);
2994 ctx->values [reg] = LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), name);
2997 case LLVMArgGsharedvtFixedVtype: {
2998 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, pindex);
3001 name = g_strdup_printf ("vtype_arg_%s", names [i]);
3003 name = g_strdup_printf ("vtype_arg_%d", i);
3005 /* Non-gsharedvt vtype argument passed by ref, the rest of the IR treats it as a vtype */
3006 g_assert (ctx->addresses [reg]);
3007 LLVMSetValueName (ctx->addresses [reg], name);
3008 LLVMBuildStore (builder, LLVMBuildLoad (builder, convert (ctx, arg, LLVMPointerType (type_to_llvm_type (ctx, ainfo->type), 0)), ""), ctx->addresses [reg]);
3011 case LLVMArgGsharedvtVariable:
3012 /* The IR treats these as variables with addresses */
3013 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindex);
3016 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, ainfo->type)), type_is_unsigned (ctx, ainfo->type));
3023 emit_volatile_store (ctx, cfg->vret_addr->dreg);
3025 emit_volatile_store (ctx, cfg->args [0]->dreg);
3026 for (i = 0; i < sig->param_count; ++i)
3027 if (!mini_type_is_vtype (sig->params [i]))
3028 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
3030 if (sig->hasthis && !cfg->rgctx_var && cfg->gshared) {
3031 LLVMValueRef this_alloc;
3034 * The exception handling code needs the location where the this argument was
3035 * stored for gshared methods. We create a separate alloca to hold it, and mark it
3036 * with the "mono.this" custom metadata to tell llvm that it needs to save its
3037 * location into the LSDA.
3039 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
3040 /* This volatile store will keep the alloca alive */
3041 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
3043 set_metadata_flag (this_alloc, "mono.this");
3046 if (cfg->rgctx_var) {
3047 LLVMValueRef rgctx_alloc, store;
3050 * We handle the rgctx arg similarly to the this pointer.
3052 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
3053 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
3054 /* This volatile store will keep the alloca alive */
3055 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
3057 set_metadata_flag (rgctx_alloc, "mono.this");
3060 /* Initialize the method if needed */
3061 if (cfg->compile_aot && ctx->llvm_only) {
3062 /* Emit a location for the initialization code */
3063 ctx->init_bb = gen_bb (ctx, "INIT_BB");
3064 ctx->inited_bb = gen_bb (ctx, "INITED_BB");
3066 LLVMBuildBr (ctx->builder, ctx->init_bb);
3067 builder = ctx->builder = create_builder (ctx);
3068 LLVMPositionBuilderAtEnd (ctx->builder, ctx->inited_bb);
3069 ctx->bblocks [cfg->bb_entry->block_num].end_bblock = ctx->inited_bb;
3072 /* Compute nesting between clauses */
3073 ctx->nested_in = (GSList**)mono_mempool_alloc0 (cfg->mempool, sizeof (GSList*) * cfg->header->num_clauses);
3074 for (i = 0; i < cfg->header->num_clauses; ++i) {
3075 for (j = 0; j < cfg->header->num_clauses; ++j) {
3076 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
3077 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
3079 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset)
3080 ctx->nested_in [i] = g_slist_prepend_mempool (cfg->mempool, ctx->nested_in [i], GINT_TO_POINTER (j));
3085 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
3086 * it needs to continue normally, or return back to the exception handling system.
3088 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
3092 if (!(bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)))
3095 clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3096 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
3097 g_hash_table_insert (ctx->clause_to_handler, GINT_TO_POINTER (clause_index), bb);
3099 if (bb->in_scount == 0) {
3102 sprintf (name, "finally_ind_bb%d", bb->block_num);
3103 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
3104 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
3106 ctx->bblocks [bb->block_num].finally_ind = val;
3108 /* Create a variable to hold the exception var */
3110 ctx->ex_var = LLVMBuildAlloca (builder, ObjRefType (), "exvar");
3114 * Create a new bblock which CALL_HANDLER/landing pads can branch to, because branching to the
3115 * LLVM bblock containing a landing pad causes problems for the
3116 * LLVM optimizer passes.
3118 sprintf (name, "BB%d_CALL_HANDLER_TARGET", bb->block_num);
3119 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
3121 ctx->builder = old_builder;
3125 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
3127 MonoCompile *cfg = ctx->cfg;
3128 LLVMValueRef *values = ctx->values;
3129 LLVMValueRef *addresses = ctx->addresses;
3130 MonoCallInst *call = (MonoCallInst*)ins;
3131 MonoMethodSignature *sig = call->signature;
3132 LLVMValueRef callee = NULL, lcall;
3134 LLVMCallInfo *cinfo;
3138 LLVMTypeRef llvm_sig;
3140 gboolean is_virtual, calli, preserveall;
3141 LLVMBuilderRef builder = *builder_ref;
3143 if ((call->signature->call_convention != MONO_CALL_DEFAULT) && !((call->signature->call_convention == MONO_CALL_C) && ctx->llvm_only)) {
3144 set_failure (ctx, "non-default callconv");
3148 cinfo = call->cinfo;
3150 if (call->rgctx_arg_reg)
3151 cinfo->rgctx_arg = TRUE;
3152 if (call->imt_arg_reg)
3153 cinfo->imt_arg = TRUE;
3155 vretaddr = (cinfo->ret.storage == LLVMArgVtypeRetAddr || cinfo->ret.storage == LLVMArgVtypeByRef || cinfo->ret.storage == LLVMArgGsharedvtFixed || cinfo->ret.storage == LLVMArgGsharedvtVariable || cinfo->ret.storage == LLVMArgGsharedvtFixedVtype);
3157 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo);
3161 is_virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE || ins->opcode == OP_RCALL_MEMBASE);
3162 calli = !call->fptr_is_patch && (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG || ins->opcode == OP_RCALL_REG);
3164 preserveall = FALSE;
3166 /* FIXME: Avoid creating duplicate methods */
3168 if (ins->flags & MONO_INST_HAS_METHOD) {
3172 if (cfg->compile_aot) {
3173 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
3175 set_failure (ctx, "can't encode patch");
3178 if (cfg->llvm_only && call->method->klass->image->assembly == ctx->module->assembly) {
3180 * Collect instructions representing the callee into a hash so they can be replaced
3181 * by the llvm method for the callee if the callee turns out to be direct
3182 * callable. Currently this only requires it to not fail llvm compilation.
3184 GSList *l = (GSList*)g_hash_table_lookup (ctx->method_to_callers, call->method);
3185 l = g_slist_prepend (l, callee);
3186 g_hash_table_insert (ctx->method_to_callers, call->method, l);
3190 static int tramp_index;
3193 name = g_strdup_printf ("tramp_%d", tramp_index);
3196 #if LLVM_API_VERSION > 100
3198 * Use our trampoline infrastructure for lazy compilation instead of llvm's.
3199 * Make all calls through a global. The address of the global will be saved in
3200 * MonoJitDomainInfo.llvm_jit_callees and updated when the method it refers to is
3203 LLVMValueRef tramp_var = g_hash_table_lookup (ctx->jit_callees, call->method);
3206 mono_create_jit_trampoline (mono_domain_get (),
3207 call->method, &error);
3208 if (!is_ok (&error)) {
3209 set_failure (ctx, mono_error_get_message (&error));
3210 mono_error_cleanup (&error);
3214 tramp_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (llvm_sig, 0), name);
3215 LLVMSetInitializer (tramp_var, LLVMConstIntToPtr (LLVMConstInt (LLVMInt64Type (), (guint64)(size_t)target, FALSE), LLVMPointerType (llvm_sig, 0)));
3216 LLVMSetLinkage (tramp_var, LLVMExternalLinkage);
3217 g_hash_table_insert (ctx->jit_callees, call->method, tramp_var);
3219 callee = LLVMBuildLoad (builder, tramp_var, "");
3222 mono_create_jit_trampoline (mono_domain_get (),
3223 call->method, &error);
3224 if (!is_ok (&error)) {
3226 set_failure (ctx, mono_error_get_message (&error));
3227 mono_error_cleanup (&error);
3231 callee = LLVMAddFunction (ctx->lmodule, name, llvm_sig);
3234 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3239 if (!cfg->llvm_only && call->method && strstr (call->method->klass->name, "AsyncVoidMethodBuilder")) {
3240 /* LLVM miscompiles async methods */
3241 set_failure (ctx, "#13734");
3246 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
3252 memset (&ji, 0, sizeof (ji));
3253 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
3254 ji.data.target = info->name;
3256 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
3258 if (cfg->compile_aot) {
3259 callee = get_callee (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
3261 set_failure (ctx, "can't encode patch");
3265 target = (gpointer)mono_icall_get_wrapper (info);
3266 callee = emit_jit_callee (ctx, "", llvm_sig, target);
3269 if (cfg->compile_aot) {
3271 if (cfg->abs_patches) {
3272 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3274 callee = get_callee (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
3276 set_failure (ctx, "can't encode patch");
3282 set_failure (ctx, "aot");
3286 #if LLVM_API_VERSION > 100
3287 if (cfg->abs_patches) {
3288 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3292 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, &error);
3293 mono_error_assert_ok (&error);
3294 callee = emit_jit_callee (ctx, "", llvm_sig, target);
3296 g_assert_not_reached ();
3299 g_assert_not_reached ();
3302 callee = LLVMAddFunction (ctx->lmodule, "", llvm_sig);
3304 if (cfg->abs_patches) {
3305 MonoJumpInfo *abs_ji = (MonoJumpInfo*)g_hash_table_lookup (cfg->abs_patches, call->fptr);
3310 * FIXME: Some trampolines might have
3311 * their own calling convention on some platforms.
3313 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE, &error);
3314 mono_error_assert_ok (&error);
3315 LLVMAddGlobalMapping (ctx->module->ee, callee, target);
3319 LLVMAddGlobalMapping (ctx->module->ee, callee, (gpointer)call->fptr);
3326 int size = sizeof (gpointer);
3329 g_assert (ins->inst_offset % size == 0);
3330 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3332 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
3334 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
3336 if (ins->flags & MONO_INST_HAS_METHOD) {
3341 * Collect and convert arguments
3343 nargs = (sig->param_count * 16) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg;
3344 len = sizeof (LLVMValueRef) * nargs;
3345 args = (LLVMValueRef*)alloca (len);
3346 memset (args, 0, len);
3347 l = call->out_ireg_args;
3349 if (call->rgctx_arg_reg) {
3350 g_assert (values [call->rgctx_arg_reg]);
3351 g_assert (cinfo->rgctx_arg_pindex < nargs);
3353 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
3354 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
3355 * it using a volatile load.
3358 if (!ctx->imt_rgctx_loc)
3359 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3360 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3361 args [cinfo->rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3363 args [cinfo->rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->module->ptr_type);
3366 if (call->imt_arg_reg) {
3367 g_assert (!ctx->llvm_only);
3368 g_assert (values [call->imt_arg_reg]);
3369 g_assert (cinfo->imt_arg_pindex < nargs);
3371 if (!ctx->imt_rgctx_loc)
3372 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->module->ptr_type, sizeof (gpointer));
3373 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->module->ptr_type), ctx->imt_rgctx_loc);
3374 args [cinfo->imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE);
3376 args [cinfo->imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->module->ptr_type);
3379 switch (cinfo->ret.storage) {
3380 case LLVMArgGsharedvtVariable: {
3381 MonoInst *var = get_vreg_to_inst (cfg, call->inst.dreg);
3383 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
3384 args [cinfo->vret_arg_pindex] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), IntPtrType ());
3386 g_assert (addresses [call->inst.dreg]);
3387 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3393 if (!addresses [call->inst.dreg])
3394 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3395 g_assert (cinfo->vret_arg_pindex < nargs);
3396 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3397 args [cinfo->vret_arg_pindex] = addresses [call->inst.dreg];
3399 args [cinfo->vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
3405 * Sometimes the same method is called with two different signatures (i.e. with and without 'this'), so
3406 * use the real callee for argument type conversion.
3408 LLVMTypeRef callee_type = LLVMGetElementType (LLVMTypeOf (callee));
3409 LLVMTypeRef *param_types = (LLVMTypeRef*)g_alloca (sizeof (LLVMTypeRef) * LLVMCountParamTypes (callee_type));
3410 LLVMGetParamTypes (callee_type, param_types);
3412 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
3415 LLVMArgInfo *ainfo = &call->cinfo->args [i];
3417 pindex = ainfo->pindex;
3419 regpair = (guint32)(gssize)(l->data);
3420 reg = regpair & 0xffffff;
3421 args [pindex] = values [reg];
3422 switch (ainfo->storage) {
3423 case LLVMArgVtypeInReg:
3424 case LLVMArgAsFpArgs: {
3428 for (j = 0; j < ainfo->ndummy_fpargs; ++j)
3429 args [pindex + j] = LLVMConstNull (LLVMDoubleType ());
3430 pindex += ainfo->ndummy_fpargs;
3432 g_assert (addresses [reg]);
3433 emit_vtype_to_args (ctx, builder, ainfo->type, addresses [reg], ainfo, args + pindex, &nargs);
3437 // FIXME: Get rid of the VMOVE
3440 case LLVMArgVtypeByVal:
3441 g_assert (addresses [reg]);
3442 args [pindex] = addresses [reg];
3444 case LLVMArgVtypeByRef: {
3445 g_assert (addresses [reg]);
3446 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3449 case LLVMArgAsIArgs:
3450 g_assert (addresses [reg]);
3451 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
3453 case LLVMArgVtypeAsScalar:
3454 g_assert_not_reached ();
3456 case LLVMArgGsharedvtFixed:
3457 case LLVMArgGsharedvtFixedVtype:
3458 g_assert (addresses [reg]);
3459 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (type_to_llvm_arg_type (ctx, ainfo->type), 0));
3461 case LLVMArgGsharedvtVariable:
3462 g_assert (addresses [reg]);
3463 args [pindex] = convert (ctx, addresses [reg], LLVMPointerType (IntPtrType (), 0));
3466 g_assert (args [pindex]);
3467 if (i == 0 && sig->hasthis)
3468 args [pindex] = convert (ctx, args [pindex], param_types [pindex]);
3470 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, ainfo->type));
3473 g_assert (pindex <= nargs);
3478 // FIXME: Align call sites
3484 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
3487 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
3489 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
3490 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
3492 /* The two can't be used together, so use only one LLVM calling conv to pass them */
3493 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
3494 if (!sig->pinvoke && !cfg->llvm_only)
3495 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
3497 mono_llvm_set_call_preserveall_cc (lcall);
3499 if (cinfo->ret.storage == LLVMArgVtypeByRef)
3500 LLVMAddInstrAttribute (lcall, 1 + cinfo->vret_arg_pindex, LLVMStructRetAttribute);
3501 if (!ctx->llvm_only && call->rgctx_arg_reg)
3502 LLVMAddInstrAttribute (lcall, 1 + cinfo->rgctx_arg_pindex, LLVMInRegAttribute);
3503 if (call->imt_arg_reg)
3504 LLVMAddInstrAttribute (lcall, 1 + cinfo->imt_arg_pindex, LLVMInRegAttribute);
3506 /* Add byval attributes if needed */
3507 for (i = 0; i < sig->param_count; ++i) {
3508 LLVMArgInfo *ainfo = &call->cinfo->args [i + sig->hasthis];
3510 if (ainfo && ainfo->storage == LLVMArgVtypeByVal)
3511 LLVMAddInstrAttribute (lcall, 1 + ainfo->pindex, LLVMByValAttribute);
3515 * Convert the result
3517 switch (cinfo->ret.storage) {
3518 case LLVMArgVtypeInReg: {
3519 LLVMValueRef regs [2];
3521 if (LLVMTypeOf (lcall) == LLVMVoidType ())
3525 if (!addresses [ins->dreg])
3526 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
3528 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
3529 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
3530 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
3531 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
3534 case LLVMArgVtypeByVal:
3535 if (!addresses [call->inst.dreg])
3536 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3537 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
3539 case LLVMArgAsIArgs:
3540 case LLVMArgFpStruct:
3541 if (!addresses [call->inst.dreg])
3542 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3543 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3545 case LLVMArgVtypeAsScalar:
3546 if (!addresses [call->inst.dreg])
3547 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
3548 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
3550 case LLVMArgVtypeRetAddr:
3551 case LLVMArgVtypeByRef:
3552 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret))) {
3553 /* Some opcodes like STOREX_MEMBASE access these by value */
3554 g_assert (addresses [call->inst.dreg]);
3555 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3558 case LLVMArgGsharedvtVariable:
3560 case LLVMArgGsharedvtFixed:
3561 case LLVMArgGsharedvtFixedVtype:
3562 values [ins->dreg] = LLVMBuildLoad (builder, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (type_to_llvm_type (ctx, sig->ret), 0), FALSE), "");
3565 if (sig->ret->type != MONO_TYPE_VOID)
3566 /* If the method returns an unsigned value, need to zext it */
3567 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
3571 *builder_ref = ctx->builder;
3575 emit_llvmonly_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3577 const char *icall_name = rethrow ? "mono_llvm_rethrow_exception" : "mono_llvm_throw_exception";
3578 LLVMValueRef callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3580 LLVMTypeRef exc_type = type_to_llvm_type (ctx, &mono_get_exception_class ()->byval_arg);
3583 LLVMTypeRef fun_sig = LLVMFunctionType1 (LLVMVoidType (), exc_type, FALSE);
3585 if (ctx->cfg->compile_aot) {
3586 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_JIT_ICALL_ADDR, icall_name);
3588 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3589 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3590 mono_memory_barrier ();
3593 ctx->module->rethrow = callee;
3595 ctx->module->throw_icall = callee;
3599 LLVMValueRef args [2];
3601 args [0] = convert (ctx, exc, exc_type);
3602 emit_call (ctx, bb, &ctx->builder, callee, args, 1);
3604 LLVMBuildUnreachable (ctx->builder);
3606 ctx->builder = create_builder (ctx);
3610 emit_throw (EmitContext *ctx, MonoBasicBlock *bb, gboolean rethrow, LLVMValueRef exc)
3612 MonoMethodSignature *throw_sig;
3613 LLVMValueRef callee, arg;
3614 const char *icall_name;
3616 callee = rethrow ? ctx->module->rethrow : ctx->module->throw_icall;
3617 icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
3620 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
3621 throw_sig->ret = &mono_get_void_class ()->byval_arg;
3622 throw_sig->params [0] = &mono_get_object_class ()->byval_arg;
3623 if (ctx->cfg->compile_aot) {
3624 callee = get_callee (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3629 * LLVM doesn't push the exception argument, so we need a different
3632 target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline");
3634 target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3636 callee = emit_jit_callee (ctx, icall_name, sig_to_llvm_sig (ctx, throw_sig), target);
3639 mono_memory_barrier ();
3640 #if LLVM_API_VERSION < 100
3642 ctx->module->rethrow = callee;
3644 ctx->module->throw_icall = callee;
3647 arg = convert (ctx, exc, type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg));
3648 emit_call (ctx, bb, &ctx->builder, callee, &arg, 1);
3652 emit_resume_eh (EmitContext *ctx, MonoBasicBlock *bb)
3654 const char *icall_name = "mono_llvm_resume_exception";
3655 LLVMValueRef callee = ctx->module->resume_eh;
3657 LLVMTypeRef fun_sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
3660 if (ctx->cfg->compile_aot) {
3661 callee = get_callee (ctx, fun_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3663 callee = LLVMAddFunction (ctx->lmodule, icall_name, fun_sig);
3664 LLVMAddGlobalMapping (ctx->module->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3665 mono_memory_barrier ();
3667 ctx->module->resume_eh = callee;
3671 emit_call (ctx, bb, &ctx->builder, callee, NULL, 0);
3673 LLVMBuildUnreachable (ctx->builder);
3675 ctx->builder = create_builder (ctx);
3679 mono_llvm_emit_clear_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3681 const char *icall_name = "mono_llvm_clear_exception";
3683 LLVMTypeRef call_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
3684 LLVMValueRef callee = NULL;
3687 if (ctx->cfg->compile_aot) {
3688 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3690 // FIXME: This is broken.
3691 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3695 g_assert (builder && callee);
3697 return LLVMBuildCall (builder, callee, NULL, 0, "");
3701 mono_llvm_emit_load_exception_call (EmitContext *ctx, LLVMBuilderRef builder)
3703 const char *icall_name = "mono_llvm_load_exception";
3705 LLVMTypeRef call_sig = LLVMFunctionType (ObjRefType (), NULL, 0, FALSE);
3706 LLVMValueRef callee = NULL;
3709 if (ctx->cfg->compile_aot) {
3710 callee = get_callee (ctx, call_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3712 // FIXME: This is broken.
3713 callee = LLVMAddFunction (ctx->lmodule, icall_name, call_sig);
3717 g_assert (builder && callee);
3719 return LLVMBuildCall (builder, callee, NULL, 0, icall_name);
3724 mono_llvm_emit_match_exception_call (EmitContext *ctx, LLVMBuilderRef builder, gint32 region_start, gint32 region_end)
3726 const char *icall_name = "mono_llvm_match_exception";
3728 ctx->builder = builder;
3730 const int num_args = 5;
3731 LLVMValueRef args [num_args];
3732 args [0] = convert (ctx, get_aotconst (ctx, MONO_PATCH_INFO_AOT_JIT_INFO, GINT_TO_POINTER (ctx->cfg->method_index)), IntPtrType ());
3733 args [1] = LLVMConstInt (LLVMInt32Type (), region_start, 0);
3734 args [2] = LLVMConstInt (LLVMInt32Type (), region_end, 0);
3735 if (ctx->cfg->rgctx_var) {
3736 LLVMValueRef rgctx_alloc = ctx->addresses [ctx->cfg->rgctx_var->dreg];
3737 g_assert (rgctx_alloc);
3738 args [3] = LLVMBuildLoad (builder, convert (ctx, rgctx_alloc, LLVMPointerType (IntPtrType (), 0)), "");
3740 args [3] = LLVMConstInt (IntPtrType (), 0, 0);
3743 args [4] = convert (ctx, ctx->this_arg, IntPtrType ());
3745 args [4] = LLVMConstInt (IntPtrType (), 0, 0);
3747 LLVMTypeRef match_sig = LLVMFunctionType5 (LLVMInt32Type (), IntPtrType (), LLVMInt32Type (), LLVMInt32Type (), IntPtrType (), IntPtrType (), FALSE);
3748 LLVMValueRef callee = ctx->module->match_exc;
3751 if (ctx->cfg->compile_aot) {
3752 ctx->builder = builder;
3753 // get_callee expects ctx->builder to be the emitting builder
3754 callee = get_callee (ctx, match_sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
3756 callee = ctx->module->match_exc = LLVMAddFunction (ctx->lmodule, icall_name, match_sig);
3757 LLVMAddGlobalMapping (ctx->module->ee, ctx->module->match_exc, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
3758 ctx->module->match_exc = callee;
3759 mono_memory_barrier ();
3763 g_assert (builder && callee);
3765 g_assert (ctx->ex_var);
3767 return LLVMBuildCall (builder, callee, args, num_args, icall_name);
3770 // FIXME: This won't work because the code-finding makes this
3772 /*#define MONO_PERSONALITY_DEBUG*/
3774 #ifdef MONO_PERSONALITY_DEBUG
3775 static const gboolean use_debug_personality = TRUE;
3776 static const char *default_personality_name = "mono_debug_personality";
3778 static const gboolean use_debug_personality = FALSE;
3779 static const char *default_personality_name = "__gxx_personality_v0";
3783 default_cpp_lpad_exc_signature (void)
3785 static gboolean inited = FALSE;
3786 static LLVMTypeRef sig;
3789 LLVMTypeRef signature [2];
3790 signature [0] = LLVMPointerType (LLVMInt8Type (), 0);
3791 signature [1] = LLVMInt32Type ();
3792 sig = LLVMStructType (signature, 2, FALSE);
3800 get_mono_personality (EmitContext *ctx)
3802 LLVMValueRef personality = NULL;
3803 static gint32 mapping_inited = FALSE;
3804 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
3806 if (!use_debug_personality) {
3807 if (ctx->cfg->compile_aot) {
3808 personality = get_intrinsic (ctx, default_personality_name);
3809 } else if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0) {
3810 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3811 LLVMAddGlobalMapping (ctx->module->ee, personality, personality);
3814 if (ctx->cfg->compile_aot) {
3815 personality = get_callee (ctx, personality_type, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name);
3817 personality = LLVMAddFunction (ctx->lmodule, default_personality_name, personality_type);
3818 LLVMAddGlobalMapping (ctx->module->ee, personality, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, default_personality_name));
3819 mono_memory_barrier ();
3823 g_assert (personality);
3827 static LLVMBasicBlockRef
3828 emit_landing_pad (EmitContext *ctx, int group_index, int group_size)
3830 MonoCompile *cfg = ctx->cfg;
3831 LLVMBuilderRef old_builder = ctx->builder;
3832 MonoExceptionClause *group_start = cfg->header->clauses + group_index;
3834 LLVMBuilderRef lpadBuilder = create_builder (ctx);
3835 ctx->builder = lpadBuilder;
3837 MonoBasicBlock *handler_bb = cfg->cil_offset_to_bb [CLAUSE_START (group_start)];
3838 g_assert (handler_bb);
3840 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3841 LLVMValueRef personality = get_mono_personality (ctx);
3842 g_assert (personality);
3844 char *bb_name = g_strdup_printf ("LPAD%d_BB", group_index);
3845 LLVMBasicBlockRef lpad_bb = gen_bb (ctx, bb_name);
3847 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3848 LLVMValueRef landing_pad = LLVMBuildLandingPad (lpadBuilder, default_cpp_lpad_exc_signature (), personality, 0, "");
3849 g_assert (landing_pad);
3851 LLVMValueRef cast = LLVMBuildBitCast (lpadBuilder, ctx->module->sentinel_exception, LLVMPointerType (LLVMInt8Type (), 0), "int8TypeInfo");
3852 LLVMAddClause (landing_pad, cast);
3854 LLVMBasicBlockRef resume_bb = gen_bb (ctx, "RESUME_BB");
3855 LLVMBuilderRef resume_builder = create_builder (ctx);
3856 ctx->builder = resume_builder;
3857 LLVMPositionBuilderAtEnd (resume_builder, resume_bb);
3859 emit_resume_eh (ctx, handler_bb);
3862 ctx->builder = lpadBuilder;
3863 LLVMPositionBuilderAtEnd (lpadBuilder, lpad_bb);
3865 gboolean finally_only = TRUE;
3867 MonoExceptionClause *group_cursor = group_start;
3869 for (int i = 0; i < group_size; i ++) {
3870 if (!(group_cursor->flags & MONO_EXCEPTION_CLAUSE_FINALLY))
3871 finally_only = FALSE;
3877 // Handle landing pad inlining
3879 if (!finally_only) {
3880 // So at each level of the exception stack we will match the exception again.
3881 // During that match, we need to compare against the handler types for the current
3882 // protected region. We send the try start and end so that we can only check against
3883 // handlers for this lexical protected region.
3884 LLVMValueRef match = mono_llvm_emit_match_exception_call (ctx, lpadBuilder, group_start->try_offset, group_start->try_offset + group_start->try_len);
3886 // if returns -1, resume
3887 LLVMValueRef switch_ins = LLVMBuildSwitch (lpadBuilder, match, resume_bb, group_size);
3889 // else move to that target bb
3890 for (int i=0; i < group_size; i++) {
3891 MonoExceptionClause *clause = group_start + i;
3892 int clause_index = clause - cfg->header->clauses;
3893 MonoBasicBlock *handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3894 g_assert (handler_bb);
3895 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3896 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
3899 int clause_index = group_start - cfg->header->clauses;
3900 MonoBasicBlock *finally_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (clause_index));
3901 g_assert (finally_bb);
3903 LLVMBuildBr (ctx->builder, ctx->bblocks [finally_bb->block_num].call_handler_target_bb);
3906 ctx->builder = old_builder;
3913 emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb)
3915 int clause_index = MONO_REGION_CLAUSE_INDEX (bb->region);
3916 MonoExceptionClause *clause = &ctx->cfg->header->clauses [clause_index];
3918 // Make exception available to catch blocks
3919 if (!(clause->flags & MONO_EXCEPTION_CLAUSE_FINALLY)) {
3920 LLVMValueRef mono_exc = mono_llvm_emit_load_exception_call (ctx, ctx->builder);
3922 g_assert (ctx->ex_var);
3923 LLVMBuildStore (ctx->builder, LLVMBuildBitCast (ctx->builder, mono_exc, ObjRefType (), ""), ctx->ex_var);
3925 if (bb->in_scount == 1) {
3926 MonoInst *exvar = bb->in_stack [0];
3927 g_assert (!ctx->values [exvar->dreg]);
3928 g_assert (ctx->ex_var);
3929 ctx->values [exvar->dreg] = LLVMBuildLoad (ctx->builder, ctx->ex_var, "save_exception");
3930 emit_volatile_store (ctx, exvar->dreg);
3933 mono_llvm_emit_clear_exception_call (ctx, ctx->builder);
3936 LLVMBuilderRef handler_builder = create_builder (ctx);
3937 LLVMBasicBlockRef target_bb = ctx->bblocks [bb->block_num].call_handler_target_bb;
3938 LLVMPositionBuilderAtEnd (handler_builder, target_bb);
3940 // Make the handler code end with a jump to cbb
3941 LLVMBuildBr (handler_builder, cbb);
3945 emit_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef builder)
3947 MonoCompile *cfg = ctx->cfg;
3948 LLVMValueRef *values = ctx->values;
3949 LLVMModuleRef lmodule = ctx->lmodule;
3950 BBInfo *bblocks = ctx->bblocks;
3952 LLVMValueRef personality;
3953 LLVMValueRef landing_pad;
3954 LLVMBasicBlockRef target_bb;
3956 static int ti_generator;
3958 LLVMValueRef type_info;
3962 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
3964 if (cfg->compile_aot) {
3965 /* Use a dummy personality function */
3966 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3967 g_assert (personality);
3969 #if LLVM_API_VERSION > 100
3970 personality = ctx->module->personality;
3972 LLVMTypeRef personality_type = LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE);
3973 personality = LLVMAddFunction (ctx->lmodule, "mono_personality", personality_type);
3974 LLVMAddFunctionAttr (personality, LLVMNoUnwindAttribute);
3975 LLVMBasicBlockRef entry_bb = LLVMAppendBasicBlock (personality, "ENTRY");
3976 LLVMBuilderRef builder2 = LLVMCreateBuilder ();
3977 LLVMPositionBuilderAtEnd (builder2, entry_bb);
3978 LLVMBuildRet (builder2, LLVMConstInt (LLVMInt32Type (), 0, FALSE));
3979 ctx->module->personality = personality;
3980 LLVMDisposeBuilder (builder2);
3983 static gint32 mapping_inited;
3985 personality = LLVMGetNamedFunction (lmodule, "mono_personality");
3987 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
3988 LLVMAddGlobalMapping (ctx->module->ee, personality, (gpointer)mono_personality);
3992 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
3994 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
3997 * Create the type info
3999 sprintf (ti_name, "type_info_%d", ti_generator);
4002 if (cfg->compile_aot) {
4003 /* decode_eh_frame () in aot-runtime.c will decode this */
4004 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4005 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4008 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
4010 LLVMSetLinkage (type_info, LLVMInternalLinkage);
4012 #if LLVM_API_VERSION > 100
4013 type_info = LLVMAddGlobal (lmodule, LLVMInt32Type (), ti_name);
4014 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
4019 * After the cfg mempool is freed, the type info will point to stale memory,
4020 * but this is not a problem, since we decode it once in exception_cb during
4023 ti = (gint32*)mono_mempool_alloc (cfg->mempool, sizeof (gint32));
4024 *(gint32*)ti = clause_index;
4026 type_info = LLVMAddGlobal (lmodule, i8ptr, ti_name);
4028 LLVMAddGlobalMapping (ctx->module->ee, type_info, ti);
4033 LLVMTypeRef members [2], ret_type;
4035 members [0] = i8ptr;
4036 members [1] = LLVMInt32Type ();
4037 ret_type = LLVMStructType (members, 2, FALSE);
4039 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
4040 LLVMAddClause (landing_pad, type_info);
4042 /* Store the exception into the exvar */
4044 LLVMBuildStore (builder, convert (ctx, LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj"), ObjRefType ()), ctx->ex_var);
4048 * LLVM throw sites are associated with a one landing pad, and LLVM generated
4049 * code expects control to be transferred to this landing pad even in the
4050 * presence of nested clauses. The landing pad needs to branch to the landing
4051 * pads belonging to nested clauses based on the selector value returned by
4052 * the landing pad instruction, which is passed to the landing pad in a
4053 * register by the EH code.
4055 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4056 g_assert (target_bb);
4059 * Branch to the correct landing pad
4061 LLVMValueRef ex_selector = LLVMBuildExtractValue (builder, landing_pad, 1, "ex_selector");
4062 LLVMValueRef switch_ins = LLVMBuildSwitch (builder, ex_selector, target_bb, 0);
4064 for (l = ctx->nested_in [clause_index]; l; l = l->next) {
4065 int nesting_clause_index = GPOINTER_TO_INT (l->data);
4066 MonoBasicBlock *handler_bb;
4068 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->clause_to_handler, GINT_TO_POINTER (nesting_clause_index));
4069 g_assert (handler_bb);
4071 g_assert (ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4072 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), nesting_clause_index, FALSE), ctx->bblocks [handler_bb->block_num].call_handler_target_bb);
4075 /* Start a new bblock which CALL_HANDLER can branch to */
4076 target_bb = bblocks [bb->block_num].call_handler_target_bb;
4078 ctx->builder = builder = create_builder (ctx);
4079 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
4081 ctx->bblocks [bb->block_num].end_bblock = target_bb;
4083 /* Store the exception into the IL level exvar */
4084 if (bb->in_scount == 1) {
4085 g_assert (bb->in_scount == 1);
4086 exvar = bb->in_stack [0];
4088 // FIXME: This is shared with filter clauses ?
4089 g_assert (!values [exvar->dreg]);
4091 g_assert (ctx->ex_var);
4092 values [exvar->dreg] = LLVMBuildLoad (builder, ctx->ex_var, "");
4093 emit_volatile_store (ctx, exvar->dreg);
4099 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
4101 MonoCompile *cfg = ctx->cfg;
4102 MonoMethodSignature *sig = ctx->sig;
4103 LLVMValueRef method = ctx->lmethod;
4104 LLVMValueRef *values = ctx->values;
4105 LLVMValueRef *addresses = ctx->addresses;
4106 LLVMCallInfo *linfo = ctx->linfo;
4107 BBInfo *bblocks = ctx->bblocks;
4109 LLVMBasicBlockRef cbb;
4110 LLVMBuilderRef builder, starting_builder;
4111 gboolean has_terminator;
4113 LLVMValueRef lhs, rhs;
4116 cbb = get_end_bb (ctx, bb);
4118 builder = create_builder (ctx);
4119 ctx->builder = builder;
4120 LLVMPositionBuilderAtEnd (builder, cbb);
4125 if (bb->flags & BB_EXCEPTION_HANDLER) {
4126 if (!ctx->llvm_only && !bblocks [bb->block_num].invoke_target) {
4127 set_failure (ctx, "handler without invokes");
4132 emit_llvmonly_handler_start (ctx, bb, cbb);
4134 emit_handler_start (ctx, bb, builder);
4137 builder = ctx->builder;
4140 has_terminator = FALSE;
4141 starting_builder = builder;
4142 for (ins = bb->code; ins; ins = ins->next) {
4143 const char *spec = LLVM_INS_INFO (ins->opcode);
4145 char dname_buf [128];
4147 emit_dbg_loc (ctx, builder, ins->cil_code);
4152 * Some steps in llc are non-linear in the size of basic blocks, see #5714.
4153 * Start a new bblock. If the llvm optimization passes merge these, we
4154 * can work around that by doing a volatile load + cond branch from
4155 * localloc-ed memory.
4157 //set_failure (ctx, "basic block too long");
4158 cbb = gen_bb (ctx, "CONT_LONG_BB");
4159 LLVMBuildBr (ctx->builder, cbb);
4160 ctx->builder = builder = create_builder (ctx);
4161 LLVMPositionBuilderAtEnd (builder, cbb);
4162 ctx->bblocks [bb->block_num].end_bblock = cbb;
4167 /* There could be instructions after a terminator, skip them */
4170 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
4171 sprintf (dname_buf, "t%d", ins->dreg);
4175 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
4176 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
4178 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) && var->opcode != OP_GSHAREDVT_ARG_REGOFFSET) {
4179 lhs = emit_volatile_load (ctx, ins->sreg1);
4181 /* It is ok for SETRET to have an uninitialized argument */
4182 if (!values [ins->sreg1] && ins->opcode != OP_SETRET) {
4183 set_failure (ctx, "sreg1");
4186 lhs = values [ins->sreg1];
4192 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
4193 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
4194 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
4195 rhs = emit_volatile_load (ctx, ins->sreg2);
4197 if (!values [ins->sreg2]) {
4198 set_failure (ctx, "sreg2");
4201 rhs = values [ins->sreg2];
4207 //mono_print_ins (ins);
4208 switch (ins->opcode) {
4211 case OP_LIVERANGE_START:
4212 case OP_LIVERANGE_END:
4215 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
4218 #if SIZEOF_VOID_P == 4
4219 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4221 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
4225 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
4229 values [ins->dreg] = LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0);
4231 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
4233 case OP_DUMMY_ICONST:
4234 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4236 case OP_DUMMY_I8CONST:
4237 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
4239 case OP_DUMMY_R8CONST:
4240 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
4243 LLVMBasicBlockRef target_bb = get_bb (ctx, ins->inst_target_bb);
4244 LLVMBuildBr (builder, target_bb);
4245 has_terminator = TRUE;
4252 LLVMBasicBlockRef new_bb;
4253 LLVMBuilderRef new_builder;
4255 // The default branch is already handled
4256 // FIXME: Handle it here
4258 /* Start new bblock */
4259 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
4260 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
4262 lhs = convert (ctx, lhs, LLVMInt32Type ());
4263 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
4264 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
4265 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
4267 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
4270 new_builder = create_builder (ctx);
4271 LLVMPositionBuilderAtEnd (new_builder, new_bb);
4272 LLVMBuildUnreachable (new_builder);
4274 has_terminator = TRUE;
4275 g_assert (!ins->next);
4281 switch (linfo->ret.storage) {
4282 case LLVMArgVtypeInReg: {
4283 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4284 LLVMValueRef val, addr, retval;
4287 retval = LLVMGetUndef (ret_type);
4289 if (!addresses [ins->sreg1]) {
4291 * The return type is an LLVM vector type, have to convert between it and the
4292 * real return type which is a struct type.
4294 g_assert (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->ret)));
4295 /* Convert to 2xi64 first */
4296 val = LLVMBuildBitCast (builder, values [ins->sreg1], LLVMVectorType (IntPtrType (), 2), "");
4298 for (i = 0; i < 2; ++i) {
4299 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4300 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildExtractElement (builder, val, LLVMConstInt (LLVMInt32Type (), i, FALSE), ""), i, "");
4302 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4306 addr = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), "");
4307 for (i = 0; i < 2; ++i) {
4308 if (linfo->ret.pair_storage [i] == LLVMArgInIReg) {
4309 LLVMValueRef indexes [2], part_addr;
4311 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4312 indexes [1] = LLVMConstInt (LLVMInt32Type (), i, FALSE);
4313 part_addr = LLVMBuildGEP (builder, addr, indexes, 2, "");
4315 retval = LLVMBuildInsertValue (builder, retval, LLVMBuildLoad (builder, part_addr, ""), i, "");
4317 g_assert (linfo->ret.pair_storage [i] == LLVMArgNone);
4321 LLVMBuildRet (builder, retval);
4324 case LLVMArgVtypeAsScalar: {
4325 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4326 LLVMValueRef retval;
4328 g_assert (addresses [ins->sreg1]);
4330 retval = LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (ret_type, 0), ""), "");
4331 LLVMBuildRet (builder, retval);
4334 case LLVMArgVtypeByVal: {
4335 LLVMValueRef retval;
4337 g_assert (addresses [ins->sreg1]);
4338 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
4339 LLVMBuildRet (builder, retval);
4342 case LLVMArgVtypeByRef: {
4343 LLVMBuildRetVoid (builder);
4346 case LLVMArgGsharedvtFixed: {
4347 LLVMTypeRef ret_type = type_to_llvm_type (ctx, sig->ret);
4348 /* The return value is in lhs, need to store to the vret argument */
4349 /* sreg1 might not be set */
4351 g_assert (cfg->vret_addr);
4352 g_assert (values [cfg->vret_addr->dreg]);
4353 LLVMBuildStore (builder, convert (ctx, lhs, ret_type), convert (ctx, values [cfg->vret_addr->dreg], LLVMPointerType (ret_type, 0)));
4355 LLVMBuildRetVoid (builder);
4358 case LLVMArgGsharedvtFixedVtype: {
4360 LLVMBuildRetVoid (builder);
4363 case LLVMArgGsharedvtVariable: {
4365 LLVMBuildRetVoid (builder);
4368 case LLVMArgVtypeRetAddr: {
4369 LLVMBuildRetVoid (builder);
4372 case LLVMArgAsIArgs:
4373 case LLVMArgFpStruct: {
4374 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
4375 LLVMValueRef retval;
4377 g_assert (addresses [ins->sreg1]);
4378 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
4379 LLVMBuildRet (builder, retval);
4383 case LLVMArgNormal: {
4384 if (!lhs || ctx->is_dead [ins->sreg1]) {
4386 * The method did not set its return value, probably because it
4387 * ends with a throw.
4390 LLVMBuildRetVoid (builder);
4392 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
4394 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
4396 has_terminator = TRUE;
4400 g_assert_not_reached ();
4409 case OP_ICOMPARE_IMM:
4410 case OP_LCOMPARE_IMM:
4411 case OP_COMPARE_IMM: {
4413 LLVMValueRef cmp, args [16];
4414 gboolean likely = (ins->flags & MONO_INST_LIKELY) != 0;
4416 if (ins->next->opcode == OP_NOP)
4419 if (ins->next->opcode == OP_BR)
4420 /* The comparison result is not needed */
4423 rel = mono_opcode_to_cond (ins->next->opcode);
4425 if (ins->opcode == OP_ICOMPARE_IMM) {
4426 lhs = convert (ctx, lhs, LLVMInt32Type ());
4427 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4429 if (ins->opcode == OP_LCOMPARE_IMM) {
4430 lhs = convert (ctx, lhs, LLVMInt64Type ());
4431 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4433 if (ins->opcode == OP_LCOMPARE) {
4434 lhs = convert (ctx, lhs, LLVMInt64Type ());
4435 rhs = convert (ctx, rhs, LLVMInt64Type ());
4437 if (ins->opcode == OP_ICOMPARE) {
4438 lhs = convert (ctx, lhs, LLVMInt32Type ());
4439 rhs = convert (ctx, rhs, LLVMInt32Type ());
4443 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4444 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
4445 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
4446 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
4449 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
4450 if (ins->opcode == OP_FCOMPARE) {
4451 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4452 } else if (ins->opcode == OP_RCOMPARE) {
4453 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4454 } else if (ins->opcode == OP_COMPARE_IMM) {
4455 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0)
4456 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
4458 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
4459 } else if (ins->opcode == OP_LCOMPARE_IMM) {
4460 if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) {
4461 /* The immediate is encoded in two fields */
4462 guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm);
4463 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), "");
4465 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), "");
4468 else if (ins->opcode == OP_COMPARE) {
4469 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
4470 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4472 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
4474 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
4478 args [1] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
4479 cmp = LLVMBuildCall (ctx->builder, get_intrinsic (ctx, "llvm.expect.i1"), args, 2, "");
4482 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
4483 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
4485 * If the target bb contains PHI instructions, LLVM requires
4486 * two PHI entries for this bblock, while we only generate one.
4487 * So convert this to an unconditional bblock. (bxc #171).
4489 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
4491 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
4493 has_terminator = TRUE;
4494 } else if (MONO_IS_SETCC (ins->next)) {
4495 sprintf (dname_buf, "t%d", ins->next->dreg);
4497 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4499 /* Add stores for volatile variables */
4500 emit_volatile_store (ctx, ins->next->dreg);
4501 } else if (MONO_IS_COND_EXC (ins->next)) {
4502 emit_cond_system_exception (ctx, bb, (const char*)ins->next->inst_p1, cmp);
4505 builder = ctx->builder;
4507 set_failure (ctx, "next");
4525 rel = mono_opcode_to_cond (ins->opcode);
4527 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
4528 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4539 rel = mono_opcode_to_cond (ins->opcode);
4541 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
4542 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
4550 gboolean empty = TRUE;
4552 /* Check that all input bblocks really branch to us */
4553 for (i = 0; i < bb->in_count; ++i) {
4554 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
4555 ins->inst_phi_args [i + 1] = -1;
4561 /* LLVM doesn't like phi instructions with zero operands */
4562 ctx->is_dead [ins->dreg] = TRUE;
4566 /* Created earlier, insert it now */
4567 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
4569 for (i = 0; i < ins->inst_phi_args [0]; i++) {
4570 int sreg1 = ins->inst_phi_args [i + 1];
4574 * Count the number of times the incoming bblock branches to us,
4575 * since llvm requires a separate entry for each.
4577 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
4578 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
4581 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
4582 if (switch_ins->inst_many_bb [j] == bb)
4589 /* Remember for later */
4590 for (j = 0; j < count; ++j) {
4591 PhiNode *node = (PhiNode*)mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
4594 node->in_bb = bb->in_bb [i];
4596 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
4606 values [ins->dreg] = lhs;
4610 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
4613 values [ins->dreg] = lhs;
4615 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
4617 * This is added by the spilling pass in case of the JIT,
4618 * but we have to do it ourselves.
4620 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
4624 case OP_MOVE_F_TO_I4: {
4625 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
4628 case OP_MOVE_I4_TO_F: {
4629 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
4632 case OP_MOVE_F_TO_I8: {
4633 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
4636 case OP_MOVE_I8_TO_F: {
4637 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
4670 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4671 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
4673 emit_div_check (ctx, builder, bb, ins, lhs, rhs);
4676 builder = ctx->builder;
4678 switch (ins->opcode) {
4681 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
4685 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
4689 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
4693 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
4697 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
4701 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
4705 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
4709 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4713 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
4717 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
4721 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
4725 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
4729 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
4733 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
4737 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4740 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4743 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4747 g_assert_not_reached ();
4754 lhs = convert (ctx, lhs, LLVMFloatType ());
4755 rhs = convert (ctx, rhs, LLVMFloatType ());
4756 switch (ins->opcode) {
4758 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
4761 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
4764 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
4767 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
4770 g_assert_not_reached ();
4779 case OP_IREM_UN_IMM:
4781 case OP_IDIV_UN_IMM:
4787 case OP_ISHR_UN_IMM:
4797 case OP_LSHR_UN_IMM:
4803 case OP_SHR_UN_IMM: {
4806 if (spec [MONO_INST_SRC1] == 'l') {
4807 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
4809 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4812 emit_div_check (ctx, builder, bb, ins, lhs, imm);
4815 builder = ctx->builder;
4817 #if SIZEOF_VOID_P == 4
4818 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
4819 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4822 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
4823 lhs = convert (ctx, lhs, IntPtrType ());
4824 imm = convert (ctx, imm, LLVMTypeOf (lhs));
4825 switch (ins->opcode) {
4829 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
4833 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
4838 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
4842 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
4844 case OP_IDIV_UN_IMM:
4845 case OP_LDIV_UN_IMM:
4846 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
4850 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
4852 case OP_IREM_UN_IMM:
4853 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
4858 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
4862 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
4866 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
4871 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
4876 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
4878 case OP_ISHR_UN_IMM:
4879 /* This is used to implement conv.u4, so the lhs could be an i8 */
4880 lhs = convert (ctx, lhs, LLVMInt32Type ());
4881 imm = convert (ctx, imm, LLVMInt32Type ());
4882 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4884 case OP_LSHR_UN_IMM:
4886 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
4889 g_assert_not_reached ();
4894 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4897 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
4900 lhs = convert (ctx, lhs, LLVMDoubleType ());
4901 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
4904 lhs = convert (ctx, lhs, LLVMFloatType ());
4905 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
4908 guint32 v = 0xffffffff;
4909 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
4913 guint64 v = 0xffffffffffffffffLL;
4914 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
4917 #if defined(TARGET_X86) || defined(TARGET_AMD64)
4919 LLVMValueRef v1, v2;
4921 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
4922 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
4923 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
4928 case OP_ICONV_TO_I1:
4929 case OP_ICONV_TO_I2:
4930 case OP_ICONV_TO_I4:
4931 case OP_ICONV_TO_U1:
4932 case OP_ICONV_TO_U2:
4933 case OP_ICONV_TO_U4:
4934 case OP_LCONV_TO_I1:
4935 case OP_LCONV_TO_I2:
4936 case OP_LCONV_TO_U1:
4937 case OP_LCONV_TO_U2:
4938 case OP_LCONV_TO_U4: {
4941 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
4943 /* Have to do two casts since our vregs have type int */
4944 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
4946 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
4948 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
4951 case OP_ICONV_TO_I8:
4952 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
4954 case OP_ICONV_TO_U8:
4955 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
4957 case OP_FCONV_TO_I4:
4958 case OP_RCONV_TO_I4:
4959 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
4961 case OP_FCONV_TO_I1:
4962 case OP_RCONV_TO_I1:
4963 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
4965 case OP_FCONV_TO_U1:
4966 case OP_RCONV_TO_U1:
4967 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildTrunc (builder, LLVMBuildFPToUI (builder, lhs, IntPtrType (), dname), LLVMInt8Type (), ""), LLVMInt32Type (), "");
4969 case OP_FCONV_TO_I2:
4970 case OP_RCONV_TO_I2:
4971 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4973 case OP_FCONV_TO_U2:
4974 case OP_RCONV_TO_U2:
4975 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
4977 case OP_RCONV_TO_U4:
4978 values [ins->dreg] = LLVMBuildFPToUI (builder, lhs, LLVMInt32Type (), dname);
4980 case OP_FCONV_TO_I8:
4981 case OP_RCONV_TO_I8:
4982 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
4985 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
4987 case OP_ICONV_TO_R8:
4988 case OP_LCONV_TO_R8:
4989 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
4991 case OP_ICONV_TO_R_UN:
4992 case OP_LCONV_TO_R_UN:
4993 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
4995 #if SIZEOF_VOID_P == 4
4998 case OP_LCONV_TO_I4:
4999 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5001 case OP_ICONV_TO_R4:
5002 case OP_LCONV_TO_R4:
5003 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
5005 values [ins->dreg] = v;
5007 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5009 case OP_FCONV_TO_R4:
5010 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
5012 values [ins->dreg] = v;
5014 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
5016 case OP_RCONV_TO_R8:
5017 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
5019 case OP_RCONV_TO_R4:
5020 values [ins->dreg] = lhs;
5023 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5026 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
5029 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
5031 case OP_LOCALLOC_IMM: {
5034 guint32 size = ins->inst_imm;
5035 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
5037 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
5039 if (ins->flags & MONO_INST_INIT) {
5040 LLVMValueRef args [5];
5043 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5044 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
5045 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
5046 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5047 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memset.p0i8.i32"), args, 5, "");
5050 values [ins->dreg] = v;
5054 LLVMValueRef v, size;
5056 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
5058 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
5060 if (ins->flags & MONO_INST_INIT) {
5061 LLVMValueRef args [5];
5064 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5066 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
5067 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5068 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memset.p0i8.i32"), args, 5, "");
5070 values [ins->dreg] = v;
5074 case OP_LOADI1_MEMBASE:
5075 case OP_LOADU1_MEMBASE:
5076 case OP_LOADI2_MEMBASE:
5077 case OP_LOADU2_MEMBASE:
5078 case OP_LOADI4_MEMBASE:
5079 case OP_LOADU4_MEMBASE:
5080 case OP_LOADI8_MEMBASE:
5081 case OP_LOADR4_MEMBASE:
5082 case OP_LOADR8_MEMBASE:
5083 case OP_LOAD_MEMBASE:
5091 LLVMValueRef base, index, addr;
5093 gboolean sext = FALSE, zext = FALSE;
5094 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5096 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5101 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
5102 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
5108 if (ins->inst_offset == 0) {
5110 } else if (ins->inst_offset % size != 0) {
5111 /* Unaligned load */
5112 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5113 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5115 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5116 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5120 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5122 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, base, dname, is_volatile, LLVM_BARRIER_NONE);
5124 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
5126 * These will signal LLVM that these loads do not alias any stores, and
5127 * they can't fail, allowing them to be hoisted out of loops.
5129 set_invariant_load_flag (values [ins->dreg]);
5130 #if LLVM_API_VERSION < 100
5131 set_metadata_flag (values [ins->dreg], "mono.nofail.load");
5136 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5138 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5139 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
5140 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
5144 case OP_STOREI1_MEMBASE_REG:
5145 case OP_STOREI2_MEMBASE_REG:
5146 case OP_STOREI4_MEMBASE_REG:
5147 case OP_STOREI8_MEMBASE_REG:
5148 case OP_STORER4_MEMBASE_REG:
5149 case OP_STORER8_MEMBASE_REG:
5150 case OP_STORE_MEMBASE_REG: {
5152 LLVMValueRef index, addr, base;
5154 gboolean sext = FALSE, zext = FALSE;
5155 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5157 if (!values [ins->inst_destbasereg]) {
5158 set_failure (ctx, "inst_destbasereg");
5162 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5164 base = values [ins->inst_destbasereg];
5165 if (ins->inst_offset % size != 0) {
5166 /* Unaligned store */
5167 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5168 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5170 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5171 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5173 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5177 case OP_STOREI1_MEMBASE_IMM:
5178 case OP_STOREI2_MEMBASE_IMM:
5179 case OP_STOREI4_MEMBASE_IMM:
5180 case OP_STOREI8_MEMBASE_IMM:
5181 case OP_STORE_MEMBASE_IMM: {
5183 LLVMValueRef index, addr, base;
5185 gboolean sext = FALSE, zext = FALSE;
5186 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5188 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5190 base = values [ins->inst_destbasereg];
5191 if (ins->inst_offset % size != 0) {
5192 /* Unaligned store */
5193 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
5194 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
5196 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5197 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5199 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), base, is_volatile);
5204 emit_load_general (ctx, bb, &builder, sizeof (gpointer), convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), lhs, "", TRUE, LLVM_BARRIER_NONE);
5206 case OP_OUTARG_VTRETADDR:
5214 case OP_VOIDCALL_MEMBASE:
5215 case OP_CALL_MEMBASE:
5216 case OP_LCALL_MEMBASE:
5217 case OP_FCALL_MEMBASE:
5218 case OP_RCALL_MEMBASE:
5219 case OP_VCALL_MEMBASE:
5220 case OP_VOIDCALL_REG:
5225 case OP_VCALL_REG: {
5226 process_call (ctx, bb, &builder, ins);
5231 LLVMValueRef indexes [2];
5232 MonoJumpInfo *tmp_ji, *ji;
5233 LLVMValueRef got_entry_addr;
5237 * FIXME: Can't allocate from the cfg mempool since that is freed if
5238 * the LLVM compile fails.
5240 tmp_ji = g_new0 (MonoJumpInfo, 1);
5241 tmp_ji->type = (MonoJumpInfoType)ins->inst_c1;
5242 tmp_ji->data.target = ins->inst_p0;
5244 ji = mono_aot_patch_info_dup (tmp_ji);
5247 if (ji->type == MONO_PATCH_INFO_ICALL_ADDR) {
5248 char *symbol = mono_aot_get_direct_call_symbol (MONO_PATCH_INFO_ICALL_ADDR_CALL, ji->data.target);
5251 * Avoid emitting a got entry for these since the method is directly called, and it might not be
5252 * resolvable at runtime using dlsym ().
5255 values [ins->dreg] = LLVMConstInt (IntPtrType (), 0, FALSE);
5260 ji->next = cfg->patch_info;
5261 cfg->patch_info = ji;
5263 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
5264 got_offset = mono_aot_get_got_offset (cfg->patch_info);
5265 ctx->module->max_got_offset = MAX (ctx->module->max_got_offset, got_offset);
5266 if (!mono_aot_is_shared_got_offset (got_offset)) {
5267 //mono_print_ji (ji);
5269 ctx->has_got_access = TRUE;
5272 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5273 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
5274 got_entry_addr = LLVMBuildGEP (builder, ctx->module->got_var, indexes, 2, "");
5276 name = get_aotconst_name (ji->type, ji->data.target, got_offset);
5277 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, name);
5279 /* Can't use this in llvmonly mode since the got slots are initialized by the methods themselves */
5280 if (!cfg->llvm_only)
5281 set_invariant_load_flag (values [ins->dreg]);
5284 case OP_NOT_REACHED:
5285 LLVMBuildUnreachable (builder);
5286 has_terminator = TRUE;
5287 g_assert (bb->block_num < cfg->max_block_num);
5288 ctx->unreachable [bb->block_num] = TRUE;
5289 /* Might have instructions after this */
5291 MonoInst *next = ins->next;
5293 * FIXME: If later code uses the regs defined by these instructions,
5294 * compilation will fail.
5296 MONO_DELETE_INS (bb, next);
5300 MonoInst *var = ins->inst_i0;
5302 if (var->opcode == OP_VTARG_ADDR) {
5303 /* The variable contains the vtype address */
5304 values [ins->dreg] = values [var->dreg];
5305 } else if (var->opcode == OP_GSHAREDVT_LOCAL) {
5306 values [ins->dreg] = emit_gsharedvt_ldaddr (ctx, var->dreg);
5308 values [ins->dreg] = addresses [var->dreg];
5313 LLVMValueRef args [1];
5315 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5316 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.sin.f64"), args, 1, dname);
5320 LLVMValueRef args [1];
5322 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5323 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.cos.f64"), args, 1, dname);
5327 LLVMValueRef args [1];
5329 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5330 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.sqrt.f64"), args, 1, dname);
5334 LLVMValueRef args [1];
5336 args [0] = convert (ctx, lhs, LLVMDoubleType ());
5337 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, "fabs"), args, 1, dname);
5351 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5352 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
5354 switch (ins->opcode) {
5357 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
5361 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
5365 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
5369 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
5372 g_assert_not_reached ();
5375 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
5378 case OP_ATOMIC_EXCHANGE_I4:
5379 case OP_ATOMIC_EXCHANGE_I8: {
5380 LLVMValueRef args [2];
5383 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
5384 t = LLVMInt32Type ();
5386 t = LLVMInt64Type ();
5388 g_assert (ins->inst_offset == 0);
5390 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5391 args [1] = convert (ctx, rhs, t);
5393 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
5396 case OP_ATOMIC_ADD_I4:
5397 case OP_ATOMIC_ADD_I8: {
5398 LLVMValueRef args [2];
5401 if (ins->opcode == OP_ATOMIC_ADD_I4)
5402 t = LLVMInt32Type ();
5404 t = LLVMInt64Type ();
5406 g_assert (ins->inst_offset == 0);
5408 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5409 args [1] = convert (ctx, rhs, t);
5410 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
5413 case OP_ATOMIC_CAS_I4:
5414 case OP_ATOMIC_CAS_I8: {
5415 LLVMValueRef args [3], val;
5418 if (ins->opcode == OP_ATOMIC_CAS_I4)
5419 t = LLVMInt32Type ();
5421 t = LLVMInt64Type ();
5423 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
5425 args [1] = convert (ctx, values [ins->sreg3], t);
5427 args [2] = convert (ctx, values [ins->sreg2], t);
5428 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
5429 /* cmpxchg returns a pair */
5430 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
5433 case OP_MEMORY_BARRIER: {
5434 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
5437 case OP_ATOMIC_LOAD_I1:
5438 case OP_ATOMIC_LOAD_I2:
5439 case OP_ATOMIC_LOAD_I4:
5440 case OP_ATOMIC_LOAD_I8:
5441 case OP_ATOMIC_LOAD_U1:
5442 case OP_ATOMIC_LOAD_U2:
5443 case OP_ATOMIC_LOAD_U4:
5444 case OP_ATOMIC_LOAD_U8:
5445 case OP_ATOMIC_LOAD_R4:
5446 case OP_ATOMIC_LOAD_R8: {
5447 #if LLVM_API_VERSION > 100
5449 gboolean sext, zext;
5451 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5452 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5453 LLVMValueRef index, addr;
5455 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5460 if (ins->inst_offset != 0) {
5461 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5462 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
5467 addr = convert (ctx, addr, LLVMPointerType (t, 0));
5469 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, lhs, dname, is_volatile, barrier);
5472 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5474 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
5477 set_failure (ctx, "atomic mono.load intrinsic");
5481 case OP_ATOMIC_STORE_I1:
5482 case OP_ATOMIC_STORE_I2:
5483 case OP_ATOMIC_STORE_I4:
5484 case OP_ATOMIC_STORE_I8:
5485 case OP_ATOMIC_STORE_U1:
5486 case OP_ATOMIC_STORE_U2:
5487 case OP_ATOMIC_STORE_U4:
5488 case OP_ATOMIC_STORE_U8:
5489 case OP_ATOMIC_STORE_R4:
5490 case OP_ATOMIC_STORE_R8: {
5492 gboolean sext, zext;
5494 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
5495 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
5496 LLVMValueRef index, addr, value, base;
5498 #if LLVM_API_VERSION < 100
5499 if (!cfg->llvm_only) {
5500 set_failure (ctx, "atomic mono.store intrinsic");
5505 if (!values [ins->inst_destbasereg]) {
5506 set_failure (ctx, "inst_destbasereg");
5510 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
5512 base = values [ins->inst_destbasereg];
5513 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
5514 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
5515 value = convert (ctx, values [ins->sreg1], t);
5517 emit_store_general (ctx, bb, &builder, size, value, addr, base, is_volatile, barrier);
5520 case OP_RELAXED_NOP: {
5521 #if defined(TARGET_AMD64) || defined(TARGET_X86)
5522 emit_call (ctx, bb, &builder, get_intrinsic (ctx, "llvm.x86.sse2.pause"), NULL, 0);
5529 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
5531 // 257 == FS segment register
5532 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
5534 // 256 == GS segment register
5535 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5538 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
5539 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
5540 /* See mono_amd64_emit_tls_get () */
5541 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
5543 // 256 == GS segment register
5544 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5545 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
5547 set_failure (ctx, "opcode tls-get");
5553 case OP_TLS_GET_REG: {
5554 #if defined(TARGET_AMD64) && defined(__linux__)
5555 // 257 == FS segment register
5556 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
5557 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt64Type ()), ptrtype, ""), "");
5558 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
5559 /* See emit_tls_get_reg () */
5560 // 256 == GS segment register
5561 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5562 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt32Type ()), ptrtype, ""), "");
5564 set_failure (ctx, "opcode tls-get");
5570 case OP_TLS_SET_REG: {
5571 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
5572 /* See emit_tls_get_reg () */
5573 // 256 == GS segment register
5574 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
5575 LLVMBuildStore (builder, convert (ctx, lhs, IntPtrType ()), LLVMBuildIntToPtr (builder, convert (ctx, rhs, LLVMInt32Type ()), ptrtype, ""));
5577 set_failure (ctx, "opcode tls-set-reg");
5582 case OP_GC_SAFE_POINT: {
5583 LLVMValueRef val, cmp, callee;
5584 LLVMBasicBlockRef poll_bb, cont_bb;
5585 static LLVMTypeRef sig;
5586 const char *icall_name = "mono_threads_state_poll";
5589 sig = LLVMFunctionType0 (LLVMVoidType (), FALSE);
5593 * mono_threads_state_poll ();
5594 * FIXME: Use a preserveall wrapper
5596 val = mono_llvm_build_load (builder, convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
5597 cmp = LLVMBuildICmp (builder, LLVMIntEQ, val, LLVMConstNull (LLVMTypeOf (val)), "");
5598 poll_bb = gen_bb (ctx, "POLL_BB");
5599 cont_bb = gen_bb (ctx, "CONT_BB");
5600 LLVMBuildCondBr (builder, cmp, cont_bb, poll_bb);
5602 ctx->builder = builder = create_builder (ctx);
5603 LLVMPositionBuilderAtEnd (builder, poll_bb);
5605 if (ctx->cfg->compile_aot) {
5606 callee = get_callee (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
5608 gpointer target = resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
5609 callee = emit_jit_callee (ctx, icall_name, sig, target);
5611 LLVMBuildCall (builder, callee, NULL, 0, "");
5612 LLVMBuildBr (builder, cont_bb);
5614 ctx->builder = builder = create_builder (ctx);
5615 LLVMPositionBuilderAtEnd (builder, cont_bb);
5616 ctx->bblocks [bb->block_num].end_bblock = cont_bb;
5624 case OP_IADD_OVF_UN:
5626 case OP_ISUB_OVF_UN:
5628 case OP_IMUL_OVF_UN:
5630 case OP_LADD_OVF_UN:
5632 case OP_LSUB_OVF_UN:
5634 case OP_LMUL_OVF_UN:
5636 LLVMValueRef args [2], val, ovf, func;
5638 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
5639 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
5640 func = get_intrinsic (ctx, ovf_op_to_intrins (ins->opcode));
5642 val = LLVMBuildCall (builder, func, args, 2, "");
5643 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
5644 ovf = LLVMBuildExtractValue (builder, val, 1, "");
5645 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
5648 builder = ctx->builder;
5654 * We currently model them using arrays. Promotion to local vregs is
5655 * disabled for them in mono_handle_global_vregs () in the LLVM case,
5656 * so we always have an entry in cfg->varinfo for them.
5657 * FIXME: Is this needed ?
5660 MonoClass *klass = ins->klass;
5661 LLVMValueRef args [5];
5665 set_failure (ctx, "!klass");
5669 if (!addresses [ins->dreg])
5670 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5671 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5672 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
5673 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5675 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5676 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5677 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memset.p0i8.i32"), args, 5, "");
5680 case OP_DUMMY_VZERO:
5683 case OP_STOREV_MEMBASE:
5684 case OP_LOADV_MEMBASE:
5686 MonoClass *klass = ins->klass;
5687 LLVMValueRef src = NULL, dst, args [5];
5688 gboolean done = FALSE;
5692 set_failure (ctx, "!klass");
5696 if (mini_is_gsharedvt_klass (klass)) {
5698 set_failure (ctx, "gsharedvt");
5702 switch (ins->opcode) {
5703 case OP_STOREV_MEMBASE:
5704 if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg &&
5705 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
5706 /* Decomposed earlier */
5707 g_assert_not_reached ();
5710 if (!addresses [ins->sreg1]) {
5712 g_assert (values [ins->sreg1]);
5713 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
5714 LLVMBuildStore (builder, values [ins->sreg1], dst);
5717 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5718 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5721 case OP_LOADV_MEMBASE:
5722 if (!addresses [ins->dreg])
5723 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5724 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
5725 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5728 if (!addresses [ins->sreg1])
5729 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
5730 if (!addresses [ins->dreg])
5731 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
5732 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
5733 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
5736 g_assert_not_reached ();
5746 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
5747 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5749 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5750 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
5751 LLVMBuildCall (builder, get_intrinsic (ctx, "llvm.memcpy.p0i8.p0i8.i32"), args, 5, "");
5754 case OP_LLVM_OUTARG_VT: {
5755 LLVMArgInfo *ainfo = (LLVMArgInfo*)ins->inst_p0;
5756 MonoType *t = mini_get_underlying_type (ins->inst_vtype);
5758 if (ainfo->storage == LLVMArgGsharedvtVariable) {
5759 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
5761 if (var && var->opcode == OP_GSHAREDVT_LOCAL) {
5762 addresses [ins->dreg] = convert (ctx, emit_gsharedvt_ldaddr (ctx, var->dreg), LLVMPointerType (IntPtrType (), 0));
5764 g_assert (addresses [ins->sreg1]);
5765 addresses [ins->dreg] = addresses [ins->sreg1];
5767 } else if (ainfo->storage == LLVMArgGsharedvtFixed) {
5768 if (!addresses [ins->sreg1]) {
5769 addresses [ins->sreg1] = build_alloca (ctx, t);
5770 g_assert (values [ins->sreg1]);
5772 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], LLVMGetElementType (LLVMTypeOf (addresses [ins->sreg1]))), addresses [ins->sreg1]);
5773 addresses [ins->dreg] = addresses [ins->sreg1];
5775 if (!addresses [ins->sreg1]) {
5776 addresses [ins->sreg1] = build_alloca (ctx, t);
5777 g_assert (values [ins->sreg1]);
5778 LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, t)), addresses [ins->sreg1]);
5780 addresses [ins->dreg] = addresses [ins->sreg1];
5784 case OP_OBJC_GET_SELECTOR: {
5785 const char *name = (const char*)ins->inst_p0;
5788 if (!ctx->module->objc_selector_to_var)
5789 ctx->module->objc_selector_to_var = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
5790 var = g_hash_table_lookup (ctx->module->objc_selector_to_var, name);
5792 LLVMValueRef indexes [16];
5794 LLVMValueRef name_var = LLVMAddGlobal (ctx->lmodule, LLVMArrayType (LLVMInt8Type (), strlen (name) + 1), "@OBJC_METH_VAR_NAME_");
5795 LLVMSetInitializer (name_var, mono_llvm_create_constant_data_array ((const uint8_t*)name, strlen (name) + 1));
5796 LLVMSetLinkage (name_var, LLVMPrivateLinkage);
5797 LLVMSetSection (name_var, "__TEXT,__objc_methname,cstring_literals");
5798 mark_as_used (ctx->module, name_var);
5800 LLVMValueRef ref_var = LLVMAddGlobal (ctx->lmodule, LLVMPointerType (LLVMInt8Type (), 0), "@OBJC_SELECTOR_REFERENCES_");
5802 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, 0);
5803 indexes [1] = LLVMConstInt (LLVMInt32Type (), 0, 0);
5804 LLVMSetInitializer (ref_var, LLVMConstGEP (name_var, indexes, 2));
5805 LLVMSetLinkage (ref_var, LLVMPrivateLinkage);
5806 LLVMSetExternallyInitialized (ref_var, TRUE);
5807 LLVMSetSection (ref_var, "__DATA, __objc_selrefs, literal_pointers, no_dead_strip");
5808 LLVMSetAlignment (ref_var, sizeof (mgreg_t));
5809 mark_as_used (ctx->module, ref_var);
5811 g_hash_table_insert (ctx->module->objc_selector_to_var, g_strdup (name), ref_var);
5815 values [ins->dreg] = LLVMBuildLoad (builder, var, "");
5822 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5824 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
5827 case OP_LOADX_MEMBASE: {
5828 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
5831 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5832 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
5835 case OP_STOREX_MEMBASE: {
5836 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
5839 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
5840 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
5847 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
5851 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
5857 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
5861 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
5865 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
5869 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
5872 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
5875 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
5878 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
5882 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
5893 LLVMValueRef v = NULL;
5895 switch (ins->opcode) {
5900 t = LLVMVectorType (LLVMInt32Type (), 4);
5901 rt = LLVMVectorType (LLVMFloatType (), 4);
5907 t = LLVMVectorType (LLVMInt64Type (), 2);
5908 rt = LLVMVectorType (LLVMDoubleType (), 2);
5911 t = LLVMInt32Type ();
5912 rt = LLVMInt32Type ();
5913 g_assert_not_reached ();
5916 lhs = LLVMBuildBitCast (builder, lhs, t, "");
5917 rhs = LLVMBuildBitCast (builder, rhs, t, "");
5918 switch (ins->opcode) {
5921 v = LLVMBuildAnd (builder, lhs, rhs, "");
5925 v = LLVMBuildOr (builder, lhs, rhs, "");
5929 v = LLVMBuildXor (builder, lhs, rhs, "");
5933 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
5936 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
5942 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntULT, lhs, rhs, "");
5943 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
5949 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntUGT, lhs, rhs, "");
5950 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
5954 LLVMValueRef cmp = LLVMBuildICmp (builder, LLVMIntSLT, lhs, rhs, "");
5955 values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, "");
5972 case OP_PADDB_SAT_UN:
5973 case OP_PADDW_SAT_UN:
5974 case OP_PSUBB_SAT_UN:
5975 case OP_PSUBW_SAT_UN:
5983 case OP_PMULW_HIGH_UN: {
5984 LLVMValueRef args [2];
5989 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
5996 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, ""), LLVMTypeOf (lhs), "");
6000 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, ""), LLVMTypeOf (lhs), "");
6008 case OP_EXTRACTX_U2:
6010 case OP_EXTRACT_U1: {
6012 gboolean zext = FALSE;
6014 t = simd_op_to_llvm_type (ins->opcode);
6016 switch (ins->opcode) {
6024 case OP_EXTRACTX_U2:
6029 t = LLVMInt32Type ();
6030 g_assert_not_reached ();
6033 lhs = LLVMBuildBitCast (builder, lhs, t, "");
6034 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
6036 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
6045 case OP_EXPAND_R8: {
6046 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6047 LLVMValueRef mask [16], v;
6050 for (i = 0; i < 16; ++i)
6051 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6053 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
6055 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6056 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
6061 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6064 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6067 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6070 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6073 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6076 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
6080 // Requires a later llvm version
6082 LLVMValueRef indexes [16];
6084 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6085 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6086 LLVMValueRef mask = LLVMConstVector (indexes, 2);
6087 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
6088 values [ins->dreg] = LLVMBuildSIToFP (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
6092 LLVMValueRef indexes [16];
6094 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6095 indexes [1] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6096 LLVMValueRef mask = LLVMConstVector (indexes, 2);
6097 LLVMValueRef shuffle = LLVMBuildShuffleVector (builder, lhs, LLVMConstNull (LLVMTypeOf (lhs)), mask, "");
6098 values [ins->dreg] = LLVMBuildFPExt (builder, shuffle, LLVMVectorType (LLVMDoubleType (), 2), dname);
6102 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMVectorType (LLVMInt32Type (), 4), dname);
6114 case OP_EXTRACT_MASK:
6121 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
6123 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
6128 LLVMRealPredicate op;
6130 switch (ins->inst_c0) {
6140 case SIMD_COMP_UNORD:
6156 g_assert_not_reached ();
6159 LLVMValueRef cmp = LLVMBuildFCmp (builder, op, lhs, rhs, "");
6160 if (ins->opcode == OP_COMPPD)
6161 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt64Type (), 2), ""), LLVMTypeOf (lhs), "");
6163 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildSExt (builder, cmp, LLVMVectorType (LLVMInt32Type (), 4), ""), LLVMTypeOf (lhs), "");
6167 /* This is only used for implementing shifts by non-immediate */
6168 values [ins->dreg] = lhs;
6179 LLVMValueRef args [3];
6182 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
6184 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6195 case OP_PSHLQ_REG: {
6196 LLVMValueRef args [3];
6199 args [1] = values [ins->sreg2];
6201 values [ins->dreg] = LLVMBuildCall (builder, get_intrinsic (ctx, simd_op_to_intrins (ins->opcode)), args, 2, dname);
6208 case OP_PSHUFLEW_LOW:
6209 case OP_PSHUFLEW_HIGH: {
6211 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
6212 int i, mask_size = 0;
6213 int imask = ins->inst_c0;
6215 /* Convert the x86 shuffle mask to LLVM's */
6216 switch (ins->opcode) {
6219 mask [0] = ((imask >> 0) & 3);
6220 mask [1] = ((imask >> 2) & 3);
6221 mask [2] = ((imask >> 4) & 3) + 4;
6222 mask [3] = ((imask >> 6) & 3) + 4;
6223 v1 = values [ins->sreg1];
6224 v2 = values [ins->sreg2];
6228 mask [0] = ((imask >> 0) & 1);
6229 mask [1] = ((imask >> 1) & 1) + 2;
6230 v1 = values [ins->sreg1];
6231 v2 = values [ins->sreg2];
6233 case OP_PSHUFLEW_LOW:
6235 mask [0] = ((imask >> 0) & 3);
6236 mask [1] = ((imask >> 2) & 3);
6237 mask [2] = ((imask >> 4) & 3);
6238 mask [3] = ((imask >> 6) & 3);
6243 v1 = values [ins->sreg1];
6244 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6246 case OP_PSHUFLEW_HIGH:
6252 mask [4] = 4 + ((imask >> 0) & 3);
6253 mask [5] = 4 + ((imask >> 2) & 3);
6254 mask [6] = 4 + ((imask >> 4) & 3);
6255 mask [7] = 4 + ((imask >> 6) & 3);
6256 v1 = values [ins->sreg1];
6257 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6261 mask [0] = ((imask >> 0) & 3);
6262 mask [1] = ((imask >> 2) & 3);
6263 mask [2] = ((imask >> 4) & 3);
6264 mask [3] = ((imask >> 6) & 3);
6265 v1 = values [ins->sreg1];
6266 v2 = LLVMGetUndef (LLVMTypeOf (v1));
6269 g_assert_not_reached ();
6271 for (i = 0; i < mask_size; ++i)
6272 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6274 values [ins->dreg] =
6275 LLVMBuildShuffleVector (builder, v1, v2,
6276 LLVMConstVector (mask_values, mask_size), dname);
6280 case OP_UNPACK_LOWB:
6281 case OP_UNPACK_LOWW:
6282 case OP_UNPACK_LOWD:
6283 case OP_UNPACK_LOWQ:
6284 case OP_UNPACK_LOWPS:
6285 case OP_UNPACK_LOWPD:
6286 case OP_UNPACK_HIGHB:
6287 case OP_UNPACK_HIGHW:
6288 case OP_UNPACK_HIGHD:
6289 case OP_UNPACK_HIGHQ:
6290 case OP_UNPACK_HIGHPS:
6291 case OP_UNPACK_HIGHPD: {
6293 LLVMValueRef mask_values [16];
6294 int i, mask_size = 0;
6295 gboolean low = FALSE;
6297 switch (ins->opcode) {
6298 case OP_UNPACK_LOWB:
6302 case OP_UNPACK_LOWW:
6306 case OP_UNPACK_LOWD:
6307 case OP_UNPACK_LOWPS:
6311 case OP_UNPACK_LOWQ:
6312 case OP_UNPACK_LOWPD:
6316 case OP_UNPACK_HIGHB:
6319 case OP_UNPACK_HIGHW:
6322 case OP_UNPACK_HIGHD:
6323 case OP_UNPACK_HIGHPS:
6326 case OP_UNPACK_HIGHQ:
6327 case OP_UNPACK_HIGHPD:
6331 g_assert_not_reached ();
6335 for (i = 0; i < (mask_size / 2); ++i) {
6337 mask [(i * 2) + 1] = mask_size + i;
6340 for (i = 0; i < (mask_size / 2); ++i) {
6341 mask [(i * 2)] = (mask_size / 2) + i;
6342 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
6346 for (i = 0; i < mask_size; ++i)
6347 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
6349 values [ins->dreg] =
6350 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
6351 LLVMConstVector (mask_values, mask_size), dname);
6356 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6357 LLVMValueRef v, val;
6359 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6360 val = LLVMConstNull (t);
6361 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6362 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
6364 values [ins->dreg] = val;
6368 case OP_DUPPS_HIGH: {
6369 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
6370 LLVMValueRef v1, v2, val;
6373 if (ins->opcode == OP_DUPPS_LOW) {
6374 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6375 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6377 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6378 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6380 val = LLVMConstNull (t);
6381 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
6382 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
6383 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
6384 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
6386 values [ins->dreg] = val;
6396 * EXCEPTION HANDLING
6398 case OP_IMPLICIT_EXCEPTION:
6399 /* This marks a place where an implicit exception can happen */
6400 if (bb->region != -1)
6401 set_failure (ctx, "implicit-exception");
6405 gboolean rethrow = (ins->opcode == OP_RETHROW);
6406 if (ctx->llvm_only) {
6407 emit_llvmonly_throw (ctx, bb, rethrow, lhs);
6408 has_terminator = TRUE;
6409 ctx->unreachable [bb->block_num] = TRUE;
6411 emit_throw (ctx, bb, rethrow, lhs);
6412 builder = ctx->builder;
6416 case OP_CALL_HANDLER: {
6418 * We don't 'call' handlers, but instead simply branch to them.
6419 * The code generated by ENDFINALLY will branch back to us.
6421 LLVMBasicBlockRef noex_bb;
6423 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
6425 bb_list = info->call_handler_return_bbs;
6428 * Set the indicator variable for the finally clause.
6430 lhs = info->finally_ind;
6432 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
6434 /* Branch to the finally clause */
6435 LLVMBuildBr (builder, info->call_handler_target_bb);
6437 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
6438 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
6440 builder = ctx->builder = create_builder (ctx);
6441 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
6443 bblocks [bb->block_num].end_bblock = noex_bb;
6446 case OP_START_HANDLER: {
6449 case OP_ENDFINALLY: {
6450 LLVMBasicBlockRef resume_bb;
6451 MonoBasicBlock *handler_bb;
6452 LLVMValueRef val, switch_ins, callee;
6456 handler_bb = (MonoBasicBlock*)g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
6457 g_assert (handler_bb);
6458 info = &bblocks [handler_bb->block_num];
6459 lhs = info->finally_ind;
6462 bb_list = info->call_handler_return_bbs;
6464 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
6466 /* Load the finally variable */
6467 val = LLVMBuildLoad (builder, lhs, "");
6469 /* Reset the variable */
6470 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
6472 /* Branch to either resume_bb, or to the bblocks in bb_list */
6473 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
6475 * The other targets are added at the end to handle OP_CALL_HANDLER
6476 * opcodes processed later.
6478 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
6480 builder = ctx->builder = create_builder (ctx);
6481 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
6483 if (ctx->llvm_only) {
6484 emit_resume_eh (ctx, bb);
6486 if (ctx->cfg->compile_aot) {
6487 callee = get_callee (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline");
6489 #if LLVM_API_VERSION > 100
6490 MonoJitICallInfo *info;
6492 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
6494 gpointer target = (void*)info->func;
6495 LLVMTypeRef icall_sig = LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE);
6496 callee = emit_jit_callee (ctx, "llvm_resume_unwind_trampoline", icall_sig, target);
6498 callee = LLVMGetNamedFunction (ctx->lmodule, "llvm_resume_unwind_trampoline");
6501 LLVMBuildCall (builder, callee, NULL, 0, "");
6502 LLVMBuildUnreachable (builder);
6505 has_terminator = TRUE;
6508 case OP_IL_SEQ_POINT:
6513 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
6514 set_failure (ctx, reason);
6522 /* Convert the value to the type required by phi nodes */
6523 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
6524 if (!values [ins->dreg])
6526 values [ins->dreg] = addresses [ins->dreg];
6528 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
6531 /* Add stores for volatile variables */
6532 if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
6533 emit_volatile_store (ctx, ins->dreg);
6539 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) {
6540 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
6543 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
6544 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
6545 LLVMBuildRetVoid (builder);
6548 if (bb == cfg->bb_entry)
6549 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
6553 * mono_llvm_check_method_supported:
6555 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
6556 * compiling a method twice.
6559 mono_llvm_check_method_supported (MonoCompile *cfg)
6566 if (cfg->method->save_lmf) {
6567 cfg->exception_message = g_strdup ("lmf");
6568 cfg->disable_llvm = TRUE;
6570 if (cfg->disable_llvm)
6574 * Nested clauses where one of the clauses is a finally clause is
6575 * not supported, because LLVM can't figure out the control flow,
6576 * probably because we resume exception handling by calling our
6577 * own function instead of using the 'resume' llvm instruction.
6579 for (i = 0; i < cfg->header->num_clauses; ++i) {
6580 for (j = 0; j < cfg->header->num_clauses; ++j) {
6581 MonoExceptionClause *clause1 = &cfg->header->clauses [i];
6582 MonoExceptionClause *clause2 = &cfg->header->clauses [j];
6584 // FIXME: Nested try clauses fail in some cases too, i.e. #37273
6585 if (i != j && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
6586 //(clause1->flags == MONO_EXCEPTION_CLAUSE_FINALLY || clause2->flags == MONO_EXCEPTION_CLAUSE_FINALLY)) {
6587 cfg->exception_message = g_strdup ("nested clauses");
6588 cfg->disable_llvm = TRUE;
6593 if (cfg->disable_llvm)
6597 if (cfg->method->dynamic) {
6598 cfg->exception_message = g_strdup ("dynamic.");
6599 cfg->disable_llvm = TRUE;
6601 if (cfg->disable_llvm)
6605 static LLVMCallInfo*
6606 get_llvm_call_info (MonoCompile *cfg, MonoMethodSignature *sig)
6608 LLVMCallInfo *linfo;
6611 if (cfg->gsharedvt && cfg->llvm_only && mini_is_gsharedvt_variable_signature (sig)) {
6615 * Gsharedvt methods have the following calling convention:
6616 * - all arguments are passed by ref, even non generic ones
6617 * - the return value is returned by ref too, using a vret
6618 * argument passed after 'this'.
6620 n = sig->param_count + sig->hasthis;
6621 linfo = (LLVMCallInfo*)mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMCallInfo) + (sizeof (LLVMArgInfo) * n));
6625 linfo->args [pindex ++].storage = LLVMArgNormal;
6627 if (sig->ret->type != MONO_TYPE_VOID) {
6628 if (mini_is_gsharedvt_variable_type (sig->ret))
6629 linfo->ret.storage = LLVMArgGsharedvtVariable;
6630 else if (mini_type_is_vtype (sig->ret))
6631 linfo->ret.storage = LLVMArgGsharedvtFixedVtype;
6633 linfo->ret.storage = LLVMArgGsharedvtFixed;
6634 linfo->vret_arg_index = pindex;
6636 linfo->ret.storage = LLVMArgNone;
6639 for (i = 0; i < sig->param_count; ++i) {
6640 if (sig->params [i]->byref)
6641 linfo->args [pindex].storage = LLVMArgNormal;
6642 else if (mini_is_gsharedvt_variable_type (sig->params [i]))
6643 linfo->args [pindex].storage = LLVMArgGsharedvtVariable;
6644 else if (mini_type_is_vtype (sig->params [i]))
6645 linfo->args [pindex].storage = LLVMArgGsharedvtFixedVtype;
6647 linfo->args [pindex].storage = LLVMArgGsharedvtFixed;
6648 linfo->args [pindex].type = sig->params [i];
6655 linfo = mono_arch_get_llvm_call_info (cfg, sig);
6656 for (i = 0; i < sig->param_count; ++i)
6657 linfo->args [i + sig->hasthis].type = sig->params [i];
6663 emit_method_inner (EmitContext *ctx);
6666 free_ctx (EmitContext *ctx)
6670 g_free (ctx->values);
6671 g_free (ctx->addresses);
6672 g_free (ctx->vreg_types);
6673 g_free (ctx->vreg_cli_types);
6674 g_free (ctx->is_dead);
6675 g_free (ctx->unreachable);
6676 g_ptr_array_free (ctx->phi_values, TRUE);
6677 g_free (ctx->bblocks);
6678 g_hash_table_destroy (ctx->region_to_handler);
6679 g_hash_table_destroy (ctx->clause_to_handler);
6680 g_hash_table_destroy (ctx->jit_callees);
6682 GHashTableIter iter;
6683 g_hash_table_iter_init (&iter, ctx->method_to_callers);
6684 while (g_hash_table_iter_next (&iter, NULL, (gpointer)&l))
6687 g_hash_table_destroy (ctx->method_to_callers);
6689 g_free (ctx->method_name);
6690 g_ptr_array_free (ctx->bblock_list, TRUE);
6692 for (l = ctx->builders; l; l = l->next) {
6693 LLVMBuilderRef builder = (LLVMBuilderRef)l->data;
6694 LLVMDisposeBuilder (builder);
6701 * mono_llvm_emit_method:
6703 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
6706 mono_llvm_emit_method (MonoCompile *cfg)
6710 gboolean is_linkonce = FALSE;
6713 /* The code below might acquire the loader lock, so use it for global locking */
6714 mono_loader_lock ();
6716 /* Used to communicate with the callbacks */
6717 mono_native_tls_set_value (current_cfg_tls_id, cfg);
6719 ctx = g_new0 (EmitContext, 1);
6721 ctx->mempool = cfg->mempool;
6724 * This maps vregs to the LLVM instruction defining them
6726 ctx->values = g_new0 (LLVMValueRef, cfg->next_vreg);
6728 * This maps vregs for volatile variables to the LLVM instruction defining their
6731 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
6732 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
6733 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
6734 ctx->phi_values = g_ptr_array_sized_new (256);
6736 * This signals whenever the vreg was defined by a phi node with no input vars
6737 * (i.e. all its input bblocks end with NOT_REACHABLE).
6739 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
6740 /* Whenever the bblock is unreachable */
6741 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
6742 ctx->bblock_list = g_ptr_array_sized_new (256);
6744 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
6745 ctx->clause_to_handler = g_hash_table_new (NULL, NULL);
6746 ctx->method_to_callers = g_hash_table_new (NULL, NULL);
6747 ctx->jit_callees = g_hash_table_new (NULL, NULL);
6748 if (cfg->compile_aot) {
6749 ctx->module = &aot_module;
6753 * Allow the linker to discard duplicate copies of wrappers, generic instances etc. by using the 'linkonce'
6754 * linkage for them. This requires the following:
6755 * - the method needs to have a unique mangled name
6756 * - llvmonly mode, since the code in aot-runtime.c would initialize got slots in the wrong aot image etc.
6758 is_linkonce = ctx->module->llvm_only && ctx->module->static_link && mono_aot_is_linkonce_method (cfg->method);
6760 method_name = mono_aot_get_mangled_method_name (cfg->method);
6762 is_linkonce = FALSE;
6765 printf ("%s %s\n", mono_method_full_name (cfg->method, 1), method_name);
6767 printf ("%s\n", mono_method_full_name (cfg->method, 1));
6771 method_name = mono_aot_get_method_name (cfg);
6772 cfg->llvm_method_name = g_strdup (method_name);
6774 init_jit_module (cfg->domain);
6775 ctx->module = (MonoLLVMModule*)domain_jit_info (cfg->domain)->llvm_module;
6776 method_name = mono_method_full_name (cfg->method, TRUE);
6778 ctx->method_name = method_name;
6779 ctx->is_linkonce = is_linkonce;
6781 #if LLVM_API_VERSION > 100
6782 if (cfg->compile_aot)
6783 ctx->lmodule = ctx->module->lmodule;
6785 ctx->lmodule = LLVMModuleCreateWithName ("jit-module");
6787 ctx->lmodule = ctx->module->lmodule;
6789 ctx->llvm_only = ctx->module->llvm_only;
6791 emit_method_inner (ctx);
6793 if (!ctx_ok (ctx)) {
6795 /* Need to add unused phi nodes as they can be referenced by other values */
6796 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (ctx->lmethod, "PHI_BB");
6797 LLVMBuilderRef builder;
6799 builder = create_builder (ctx);
6800 LLVMPositionBuilderAtEnd (builder, phi_bb);
6802 for (i = 0; i < ctx->phi_values->len; ++i) {
6803 LLVMValueRef v = (LLVMValueRef)g_ptr_array_index (ctx->phi_values, i);
6804 if (LLVMGetInstructionParent (v) == NULL)
6805 LLVMInsertIntoBuilder (builder, v);
6808 LLVMDeleteFunction (ctx->lmethod);
6814 mono_native_tls_set_value (current_cfg_tls_id, NULL);
6816 mono_loader_unlock ();
6820 emit_method_inner (EmitContext *ctx)
6822 MonoCompile *cfg = ctx->cfg;
6823 MonoMethodSignature *sig;
6825 LLVMTypeRef method_type;
6826 LLVMValueRef method = NULL;
6827 LLVMValueRef *values = ctx->values;
6828 int i, max_block_num, bb_index;
6829 gboolean last = FALSE;
6830 LLVMCallInfo *linfo;
6831 LLVMModuleRef lmodule = ctx->lmodule;
6833 GPtrArray *bblock_list = ctx->bblock_list;
6834 MonoMethodHeader *header;
6835 MonoExceptionClause *clause;
6838 if (cfg->gsharedvt && !cfg->llvm_only) {
6839 set_failure (ctx, "gsharedvt");
6845 static int count = 0;
6848 if (g_getenv ("LLVM_COUNT")) {
6849 if (count == atoi (g_getenv ("LLVM_COUNT"))) {
6850 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
6854 if (count > atoi (g_getenv ("LLVM_COUNT"))) {
6855 set_failure (ctx, "count");
6862 sig = mono_method_signature (cfg->method);
6865 linfo = get_llvm_call_info (cfg, sig);
6871 linfo->rgctx_arg = TRUE;
6872 ctx->method_type = method_type = sig_to_llvm_sig_full (ctx, sig, linfo);
6876 method = LLVMAddFunction (lmodule, ctx->method_name, method_type);
6877 ctx->lmethod = method;
6879 if (!cfg->llvm_only)
6880 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
6881 LLVMSetLinkage (method, LLVMPrivateLinkage);
6883 LLVMAddFunctionAttr (method, LLVMUWTable);
6885 if (cfg->compile_aot) {
6886 LLVMSetLinkage (method, LLVMInternalLinkage);
6887 if (ctx->module->external_symbols) {
6888 LLVMSetLinkage (method, LLVMExternalLinkage);
6889 LLVMSetVisibility (method, LLVMHiddenVisibility);
6891 if (ctx->is_linkonce) {
6892 LLVMSetLinkage (method, LLVMLinkOnceAnyLinkage);
6893 LLVMSetVisibility (method, LLVMDefaultVisibility);
6896 #if LLVM_API_VERSION > 100
6897 LLVMSetLinkage (method, LLVMExternalLinkage);
6899 LLVMSetLinkage (method, LLVMPrivateLinkage);
6903 if (cfg->method->save_lmf && !cfg->llvm_only) {
6904 set_failure (ctx, "lmf");
6908 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE && !cfg->llvm_only) {
6909 set_failure (ctx, "pinvoke signature");
6913 header = cfg->header;
6914 for (i = 0; i < header->num_clauses; ++i) {
6915 clause = &header->clauses [i];
6916 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) {
6917 set_failure (ctx, "non-finally/catch clause.");
6921 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING) || cfg->no_inline)
6922 /* We can't handle inlined methods with clauses */
6923 LLVMAddFunctionAttr (method, LLVMNoInlineAttribute);
6925 if (linfo->rgctx_arg) {
6926 ctx->rgctx_arg = LLVMGetParam (method, linfo->rgctx_arg_pindex);
6927 ctx->rgctx_arg_pindex = linfo->rgctx_arg_pindex;
6929 * We mark the rgctx parameter with the inreg attribute, which is mapped to
6930 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
6931 * CC_X86_64_Mono in X86CallingConv.td.
6933 if (!ctx->llvm_only)
6934 LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute);
6935 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
6937 ctx->rgctx_arg_pindex = -1;
6939 if (cfg->vret_addr) {
6940 values [cfg->vret_addr->dreg] = LLVMGetParam (method, linfo->vret_arg_pindex);
6941 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
6942 if (linfo->ret.storage == LLVMArgVtypeByRef) {
6943 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMStructRetAttribute);
6944 LLVMAddAttribute (LLVMGetParam (method, linfo->vret_arg_pindex), LLVMNoAliasAttribute);
6949 ctx->this_arg_pindex = linfo->this_arg_pindex;
6950 ctx->this_arg = LLVMGetParam (method, linfo->this_arg_pindex);
6951 values [cfg->args [0]->dreg] = ctx->this_arg;
6952 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
6955 names = g_new (char *, sig->param_count);
6956 mono_method_get_param_names (cfg->method, (const char **) names);
6958 /* Set parameter names/attributes */
6959 for (i = 0; i < sig->param_count; ++i) {
6960 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
6962 int pindex = ainfo->pindex + ainfo->ndummy_fpargs;
6965 for (j = 0; j < ainfo->ndummy_fpargs; ++j) {
6966 name = g_strdup_printf ("dummy_%d_%d", i, j);
6967 LLVMSetValueName (LLVMGetParam (method, ainfo->pindex + j), name);
6971 if (ainfo->storage == LLVMArgVtypeInReg && ainfo->pair_storage [0] == LLVMArgNone && ainfo->pair_storage [1] == LLVMArgNone)
6974 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex);
6975 if (ainfo->storage == LLVMArgGsharedvtFixed || ainfo->storage == LLVMArgGsharedvtFixedVtype) {
6976 if (names [i] && names [i][0] != '\0')
6977 name = g_strdup_printf ("p_arg_%s", names [i]);
6979 name = g_strdup_printf ("p_arg_%d", i);
6981 if (names [i] && names [i][0] != '\0')
6982 name = g_strdup_printf ("arg_%s", names [i]);
6984 name = g_strdup_printf ("arg_%d", i);
6986 LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name);
6988 if (ainfo->storage == LLVMArgVtypeByVal)
6989 LLVMAddAttribute (LLVMGetParam (method, pindex), LLVMByValAttribute);
6991 if (ainfo->storage == LLVMArgVtypeByRef) {
6993 cfg->args [i + sig->hasthis]->opcode = OP_VTARG_ADDR;
6998 if (ctx->module->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
6999 ctx->minfo = mono_debug_lookup_method (cfg->method);
7000 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, ctx->method_name);
7004 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
7005 max_block_num = MAX (max_block_num, bb->block_num);
7006 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
7008 /* Add branches between non-consecutive bblocks */
7009 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7010 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
7011 bb->next_bb != bb->last_ins->inst_false_bb) {
7013 MonoInst *inst = (MonoInst*)mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
7014 inst->opcode = OP_BR;
7015 inst->inst_target_bb = bb->last_ins->inst_false_bb;
7016 mono_bblock_add_inst (bb, inst);
7021 * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR
7022 * was later optimized away, so clear these flags, and add them back for the still
7023 * present OP_LDADDR instructions.
7025 for (i = 0; i < cfg->next_vreg; ++i) {
7028 ins = get_vreg_to_inst (cfg, i);
7029 if (ins && ins != cfg->rgctx_var)
7030 ins->flags &= ~MONO_INST_INDIRECT;
7034 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
7036 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7038 LLVMBuilderRef builder;
7040 char dname_buf[128];
7042 builder = create_builder (ctx);
7044 for (ins = bb->code; ins; ins = ins->next) {
7045 switch (ins->opcode) {
7050 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, &ins->klass->byval_arg));
7055 if (ins->opcode == OP_VPHI) {
7056 /* Treat valuetype PHI nodes as operating on the address itself */
7057 g_assert (ins->klass);
7058 phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
7062 * Have to precreate these, as they can be referenced by
7063 * earlier instructions.
7065 sprintf (dname_buf, "t%d", ins->dreg);
7067 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
7069 if (ins->opcode == OP_VPHI)
7070 ctx->addresses [ins->dreg] = values [ins->dreg];
7072 g_ptr_array_add (ctx->phi_values, values [ins->dreg]);
7075 * Set the expected type of the incoming arguments since these have
7076 * to have the same type.
7078 for (i = 0; i < ins->inst_phi_args [0]; i++) {
7079 int sreg1 = ins->inst_phi_args [i + 1];
7082 ctx->vreg_types [sreg1] = phi_type;
7087 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
7096 * Create an ordering for bblocks, use the depth first order first, then
7097 * put the exception handling bblocks last.
7099 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
7100 bb = cfg->bblocks [bb_index];
7101 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
7102 g_ptr_array_add (bblock_list, bb);
7103 bblocks [bb->block_num].added = TRUE;
7107 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7108 if (!bblocks [bb->block_num].added)
7109 g_ptr_array_add (bblock_list, bb);
7113 * Second pass: generate code.
7116 LLVMBuilderRef entry_builder = create_builder (ctx);
7117 LLVMBasicBlockRef entry_bb = get_bb (ctx, cfg->bb_entry);
7118 LLVMPositionBuilderAtEnd (entry_builder, entry_bb);
7119 emit_entry_bb (ctx, entry_builder);
7121 // Make landing pads first
7122 ctx->exc_meta = g_hash_table_new_full (NULL, NULL, NULL, NULL);
7124 if (ctx->llvm_only) {
7125 size_t group_index = 0;
7126 while (group_index < cfg->header->num_clauses) {
7128 size_t cursor = group_index;
7129 while (cursor < cfg->header->num_clauses &&
7130 CLAUSE_START (&cfg->header->clauses [cursor]) == CLAUSE_START (&cfg->header->clauses [group_index]) &&
7131 CLAUSE_END (&cfg->header->clauses [cursor]) == CLAUSE_END (&cfg->header->clauses [group_index])) {
7136 LLVMBasicBlockRef lpad_bb = emit_landing_pad (ctx, group_index, count);
7137 intptr_t key = CLAUSE_END (&cfg->header->clauses [group_index]);
7138 g_hash_table_insert (ctx->exc_meta, (gpointer)key, lpad_bb);
7140 group_index = cursor;
7144 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
7145 bb = (MonoBasicBlock*)g_ptr_array_index (bblock_list, bb_index);
7147 // Prune unreachable mono BBs.
7148 if (!(bb == cfg->bb_entry || bb->in_count > 0))
7151 process_bb (ctx, bb);
7155 g_hash_table_destroy (ctx->exc_meta);
7157 mono_memory_barrier ();
7159 /* Add incoming phi values */
7160 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7161 GSList *l, *ins_list;
7163 ins_list = bblocks [bb->block_num].phi_nodes;
7165 for (l = ins_list; l; l = l->next) {
7166 PhiNode *node = (PhiNode*)l->data;
7167 MonoInst *phi = node->phi;
7168 int sreg1 = node->sreg;
7169 LLVMBasicBlockRef in_bb;
7174 in_bb = get_end_bb (ctx, node->in_bb);
7176 if (ctx->unreachable [node->in_bb->block_num])
7179 if (!values [sreg1]) {
7180 /* Can happen with values in EH clauses */
7181 set_failure (ctx, "incoming phi sreg1");
7185 if (phi->opcode == OP_VPHI) {
7186 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
7187 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
7189 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg])) {
7190 set_failure (ctx, "incoming phi arg type mismatch");
7193 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
7194 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
7199 /* Nullify empty phi instructions */
7200 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7201 GSList *l, *ins_list;
7203 ins_list = bblocks [bb->block_num].phi_nodes;
7205 for (l = ins_list; l; l = l->next) {
7206 PhiNode *node = (PhiNode*)l->data;
7207 MonoInst *phi = node->phi;
7208 LLVMValueRef phi_ins = values [phi->dreg];
7211 /* Already removed */
7214 if (LLVMCountIncoming (phi_ins) == 0) {
7215 mono_llvm_replace_uses_of (phi_ins, LLVMConstNull (LLVMTypeOf (phi_ins)));
7216 LLVMInstructionEraseFromParent (phi_ins);
7217 values [phi->dreg] = NULL;
7222 /* Create the SWITCH statements for ENDFINALLY instructions */
7223 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
7224 BBInfo *info = &bblocks [bb->block_num];
7226 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
7227 LLVMValueRef switch_ins = (LLVMValueRef)l->data;
7228 GSList *bb_list = info->call_handler_return_bbs;
7230 for (i = 0; i < g_slist_length (bb_list); ++i)
7231 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), (LLVMBasicBlockRef)(g_slist_nth (bb_list, i)->data));
7235 /* Initialize the method if needed */
7236 if (cfg->compile_aot && ctx->llvm_only) {
7237 // FIXME: Add more shared got entries
7238 ctx->builder = create_builder (ctx);
7239 LLVMPositionBuilderAtEnd (ctx->builder, ctx->init_bb);
7241 ctx->module->max_method_idx = MAX (ctx->module->max_method_idx, cfg->method_index);
7243 // FIXME: beforefieldinit
7244 if (ctx->has_got_access || mono_class_get_cctor (cfg->method->klass)) {
7246 * linkonce methods shouldn't have initialization,
7247 * because they might belong to assemblies which
7248 * haven't been loaded yet.
7250 g_assert (!ctx->is_linkonce);
7251 emit_init_method (ctx);
7253 LLVMBuildBr (ctx->builder, ctx->inited_bb);
7257 if (cfg->llvm_only) {
7258 GHashTableIter iter;
7260 GSList *callers, *l, *l2;
7263 * Add the contents of ctx->method_to_callers to module->method_to_callers.
7264 * We can't do this earlier, as it contains llvm instructions which can be
7265 * freed if compilation fails.
7266 * FIXME: Get rid of this when all methods can be llvm compiled.
7268 g_hash_table_iter_init (&iter, ctx->method_to_callers);
7269 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
7270 for (l = callers; l; l = l->next) {
7271 l2 = (GSList*)g_hash_table_lookup (ctx->module->method_to_callers, method);
7272 l2 = g_slist_prepend (l2, l->data);
7273 g_hash_table_insert (ctx->module->method_to_callers, method, l2);
7278 if (cfg->verbose_level > 1)
7279 mono_llvm_dump_value (method);
7281 if (cfg->compile_aot && !cfg->llvm_only)
7282 mark_as_used (ctx->module, method);
7284 if (!cfg->llvm_only) {
7285 LLVMValueRef md_args [16];
7286 LLVMValueRef md_node;
7289 if (cfg->compile_aot)
7290 method_index = mono_aot_get_method_index (cfg->orig_method);
7293 md_args [0] = LLVMMDString (ctx->method_name, strlen (ctx->method_name));
7294 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
7295 md_node = LLVMMDNode (md_args, 2);
7296 LLVMAddNamedMetadataOperand (lmodule, "mono.function_indexes", md_node);
7297 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
7300 if (cfg->compile_aot) {
7301 /* Don't generate native code, keep the LLVM IR */
7302 if (cfg->verbose_level)
7303 printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), ctx->method_name);
7305 #if LLVM_API_VERSION < 100
7306 /* VerifyFunction can't handle some of the debug info created by DIBuilder in llvm 3.9 */
7307 int err = LLVMVerifyFunction(ctx->lmethod, LLVMPrintMessageAction);
7308 g_assert (err == 0);
7311 //LLVMVerifyFunction(method, 0);
7312 #if LLVM_API_VERSION > 100
7313 MonoDomain *domain = mono_domain_get ();
7314 MonoJitDomainInfo *domain_info;
7315 int nvars = g_hash_table_size (ctx->jit_callees);
7316 LLVMValueRef *callee_vars = g_new0 (LLVMValueRef, nvars);
7317 gpointer *callee_addrs = g_new0 (gpointer, nvars);
7318 GHashTableIter iter;
7324 * Compute the addresses of the LLVM globals pointing to the
7325 * methods called by the current method. Pass it to the trampoline
7326 * code so it can update them after their corresponding method was
7329 g_hash_table_iter_init (&iter, ctx->jit_callees);
7331 while (g_hash_table_iter_next (&iter, NULL, (void**)&var))
7332 callee_vars [i ++] = var;
7334 cfg->native_code = mono_llvm_compile_method (ctx->module->mono_ee, ctx->lmethod, nvars, callee_vars, callee_addrs, &eh_frame);
7336 decode_llvm_eh_info (ctx, eh_frame);
7338 mono_domain_lock (domain);
7339 domain_info = domain_jit_info (domain);
7340 if (!domain_info->llvm_jit_callees)
7341 domain_info->llvm_jit_callees = g_hash_table_new (NULL, NULL);
7342 g_hash_table_iter_init (&iter, ctx->jit_callees);
7344 while (g_hash_table_iter_next (&iter, (void**)&callee, (void**)&var)) {
7345 GSList *addrs = g_hash_table_lookup (domain_info->llvm_jit_callees, callee);
7346 addrs = g_slist_prepend (addrs, callee_addrs [i]);
7347 g_hash_table_insert (domain_info->llvm_jit_callees, callee, addrs);
7350 mono_domain_unlock (domain);
7352 mono_llvm_optimize_method (ctx->module->mono_ee, ctx->lmethod);
7354 if (cfg->verbose_level > 1)
7355 mono_llvm_dump_value (ctx->lmethod);
7357 cfg->native_code = (unsigned char*)LLVMGetPointerToGlobal (ctx->module->ee, ctx->lmethod);
7359 /* Set by emit_cb */
7360 g_assert (cfg->code_len);
7364 if (ctx->module->method_to_lmethod)
7365 g_hash_table_insert (ctx->module->method_to_lmethod, cfg->method, ctx->lmethod);
7366 if (ctx->module->idx_to_lmethod)
7367 g_hash_table_insert (ctx->module->idx_to_lmethod, GINT_TO_POINTER (cfg->method_index), ctx->lmethod);
7369 if (ctx->llvm_only && cfg->orig_method->klass->valuetype && !(cfg->orig_method->flags & METHOD_ATTRIBUTE_STATIC))
7370 emit_unbox_tramp (ctx, ctx->method_name, ctx->method_type, ctx->lmethod, cfg->method_index);
7374 * mono_llvm_create_vars:
7376 * Same as mono_arch_create_vars () for LLVM.
7379 mono_llvm_create_vars (MonoCompile *cfg)
7381 MonoMethodSignature *sig;
7383 sig = mono_method_signature (cfg->method);
7384 if (cfg->gsharedvt && cfg->llvm_only) {
7385 if (mini_is_gsharedvt_variable_signature (sig) && sig->ret->type != MONO_TYPE_VOID) {
7386 cfg->vret_addr = mono_compile_create_var (cfg, &mono_get_intptr_class ()->byval_arg, OP_ARG);
7387 if (G_UNLIKELY (cfg->verbose_level > 1)) {
7388 printf ("vret_addr = ");
7389 mono_print_ins (cfg->vret_addr);
7393 mono_arch_create_vars (cfg);
7398 * mono_llvm_emit_call:
7400 * Same as mono_arch_emit_call () for LLVM.
7403 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
7406 MonoMethodSignature *sig;
7407 int i, n, stack_size;
7412 sig = call->signature;
7413 n = sig->param_count + sig->hasthis;
7415 call->cinfo = get_llvm_call_info (cfg, sig);
7417 if (cfg->disable_llvm)
7420 if (sig->call_convention == MONO_CALL_VARARG) {
7421 cfg->exception_message = g_strdup ("varargs");
7422 cfg->disable_llvm = TRUE;
7425 for (i = 0; i < n; ++i) {
7428 ainfo = call->cinfo->args + i;
7430 in = call->args [i];
7432 /* Simply remember the arguments */
7433 switch (ainfo->storage) {
7434 case LLVMArgNormal: {
7435 MonoType *t = (sig->hasthis && i == 0) ? &mono_get_intptr_class ()->byval_arg : ainfo->type;
7438 opcode = mono_type_to_regmove (cfg, t);
7439 if (opcode == OP_FMOVE) {
7440 MONO_INST_NEW (cfg, ins, OP_FMOVE);
7441 ins->dreg = mono_alloc_freg (cfg);
7442 } else if (opcode == OP_LMOVE) {
7443 MONO_INST_NEW (cfg, ins, OP_LMOVE);
7444 ins->dreg = mono_alloc_lreg (cfg);
7445 } else if (opcode == OP_RMOVE) {
7446 MONO_INST_NEW (cfg, ins, OP_RMOVE);
7447 ins->dreg = mono_alloc_freg (cfg);
7449 MONO_INST_NEW (cfg, ins, OP_MOVE);
7450 ins->dreg = mono_alloc_ireg (cfg);
7452 ins->sreg1 = in->dreg;
7455 case LLVMArgVtypeByVal:
7456 case LLVMArgVtypeByRef:
7457 case LLVMArgVtypeInReg:
7458 case LLVMArgVtypeAsScalar:
7459 case LLVMArgAsIArgs:
7460 case LLVMArgAsFpArgs:
7461 case LLVMArgGsharedvtVariable:
7462 case LLVMArgGsharedvtFixed:
7463 case LLVMArgGsharedvtFixedVtype:
7464 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
7465 ins->dreg = mono_alloc_ireg (cfg);
7466 ins->sreg1 = in->dreg;
7467 ins->inst_p0 = mono_mempool_alloc0 (cfg->mempool, sizeof (LLVMArgInfo));
7468 memcpy (ins->inst_p0, ainfo, sizeof (LLVMArgInfo));
7469 ins->inst_vtype = ainfo->type;
7470 ins->klass = mono_class_from_mono_type (ainfo->type);
7473 cfg->exception_message = g_strdup ("ainfo->storage");
7474 cfg->disable_llvm = TRUE;
7478 if (!cfg->disable_llvm) {
7479 MONO_ADD_INS (cfg->cbb, ins);
7480 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
7485 static unsigned char*
7486 alloc_cb (LLVMValueRef function, int size)
7490 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7494 return (unsigned char*)mono_domain_code_reserve (cfg->domain, size);
7496 return (unsigned char*)mono_domain_code_reserve (mono_domain_get (), size);
7501 emitted_cb (LLVMValueRef function, void *start, void *end)
7505 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7507 cfg->code_len = (guint8*)end - (guint8*)start;
7511 exception_cb (void *data)
7514 MonoJitExceptionInfo *ei;
7515 guint32 ei_len, i, j, nested_len, nindex;
7516 gpointer *type_info;
7517 int this_reg, this_offset;
7519 cfg = (MonoCompile*)mono_native_tls_get_value (current_cfg_tls_id);
7523 * data points to a DWARF FDE structure, convert it to our unwind format and
7525 * An alternative would be to save it directly, and modify our unwinder to work
7528 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
7529 if (cfg->verbose_level > 1)
7530 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
7532 /* Count nested clauses */
7534 for (i = 0; i < ei_len; ++i) {
7535 gint32 cindex1 = *(gint32*)type_info [i];
7536 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
7538 for (j = 0; j < cfg->header->num_clauses; ++j) {
7540 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
7542 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7548 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
7549 cfg->llvm_ex_info_len = ei_len + nested_len;
7550 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
7551 /* Fill the rest of the information from the type info */
7552 for (i = 0; i < ei_len; ++i) {
7553 gint32 clause_index = *(gint32*)type_info [i];
7554 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
7556 cfg->llvm_ex_info [i].flags = clause->flags;
7557 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
7558 cfg->llvm_ex_info [i].clause_index = clause_index;
7562 * For nested clauses, the LLVM produced exception info associates the try interval with
7563 * the innermost handler, while mono expects it to be associated with all nesting clauses.
7564 * So add new clauses which use the IL info (catch class etc.) from the nesting clause,
7565 * and everything else from the nested clause.
7568 for (i = 0; i < ei_len; ++i) {
7569 gint32 cindex1 = *(gint32*)type_info [i];
7570 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
7572 for (j = 0; j < cfg->header->num_clauses; ++j) {
7574 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
7575 MonoJitExceptionInfo *nesting_ei, *nested_ei;
7577 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
7578 /* clause1 is the nested clause */
7579 nested_ei = &cfg->llvm_ex_info [i];
7580 nesting_ei = &cfg->llvm_ex_info [nindex];
7583 memcpy (nesting_ei, nested_ei, sizeof (MonoJitExceptionInfo));
7585 nesting_ei->flags = clause2->flags;
7586 nesting_ei->data.catch_class = clause2->data.catch_class;
7587 nesting_ei->clause_index = cindex2;
7591 g_assert (nindex == ei_len + nested_len);
7592 cfg->llvm_this_reg = this_reg;
7593 cfg->llvm_this_offset = this_offset;
7595 /* type_info [i] is cfg mempool allocated, no need to free it */
7601 #if LLVM_API_VERSION > 100
7603 * decode_llvm_eh_info:
7605 * Decode the EH table emitted by llvm in jit mode, and store
7606 * the result into cfg.
7609 decode_llvm_eh_info (EmitContext *ctx, gpointer eh_frame)
7611 MonoCompile *cfg = ctx->cfg;
7614 MonoLLVMFDEInfo info;
7615 MonoJitExceptionInfo *ei;
7616 guint8 *p = eh_frame;
7617 int version, fde_count, fde_offset;
7618 guint32 ei_len, i, nested_len;
7619 gpointer *type_info;
7623 * Decode the one element EH table emitted by the MonoException class
7627 /* Similar to decode_llvm_mono_eh_frame () in aot-runtime.c */
7630 g_assert (version == 3);
7633 p = (guint8 *)ALIGN_PTR_TO (p, 4);
7635 fde_count = *(guint32*)p;
7639 g_assert (fde_count <= 2);
7641 /* The first entry is the real method */
7642 g_assert (table [0] == 1);
7643 fde_offset = table [1];
7644 table += fde_count * 2;
7646 cfg->code_len = table [0];
7647 fde_len = table [1] - fde_offset;
7650 fde = (guint8*)eh_frame + fde_offset;
7651 cie = (guint8*)table;
7653 mono_unwind_decode_llvm_mono_fde (fde, fde_len, cie, cfg->native_code, &info);
7655 cfg->encoded_unwind_ops = info.unw_info;
7656 cfg->encoded_unwind_ops_len = info.unw_info_len;
7657 if (cfg->verbose_level > 1)
7658 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
7659 if (info.this_reg != -1) {
7660 cfg->llvm_this_reg = info.this_reg;
7661 cfg->llvm_this_offset = info.this_offset;
7665 ei_len = info.ex_info_len;
7666 type_info = info.type_info;
7668 // Nested clauses are currently disabled
7671 cfg->llvm_ex_info = (MonoJitExceptionInfo*)mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
7672 cfg->llvm_ex_info_len = ei_len + nested_len;
7673 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
7674 /* Fill the rest of the information from the type info */
7675 for (i = 0; i < ei_len; ++i) {
7676 gint32 clause_index = *(gint32*)type_info [i];
7677 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
7679 cfg->llvm_ex_info [i].flags = clause->flags;
7680 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
7681 cfg->llvm_ex_info [i].clause_index = clause_index;
7687 dlsym_cb (const char *name, void **symbol)
7693 if (!strcmp (name, "__bzero")) {
7694 *symbol = (void*)bzero;
7696 current = mono_dl_open (NULL, 0, NULL);
7699 err = mono_dl_symbol (current, name, symbol);
7701 mono_dl_close (current);
7703 #ifdef MONO_ARCH_HAVE_CREATE_LLVM_NATIVE_THUNK
7704 *symbol = (char*)mono_arch_create_llvm_native_thunk (mono_domain_get (), (guint8*)(*symbol));
7710 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
7712 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
7716 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
7718 LLVMTypeRef param_types [4];
7720 param_types [0] = param_type1;
7721 param_types [1] = param_type2;
7723 AddFunc (module, name, ret_type, param_types, 2);
7729 INTRINS_SADD_OVF_I32,
7730 INTRINS_UADD_OVF_I32,
7731 INTRINS_SSUB_OVF_I32,
7732 INTRINS_USUB_OVF_I32,
7733 INTRINS_SMUL_OVF_I32,
7734 INTRINS_UMUL_OVF_I32,
7735 INTRINS_SADD_OVF_I64,
7736 INTRINS_UADD_OVF_I64,
7737 INTRINS_SSUB_OVF_I64,
7738 INTRINS_USUB_OVF_I64,
7739 INTRINS_SMUL_OVF_I64,
7740 INTRINS_UMUL_OVF_I64,
7747 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7748 INTRINS_SSE_PMOVMSKB,
7749 INTRINS_SSE_PSRLI_W,
7750 INTRINS_SSE_PSRAI_W,
7751 INTRINS_SSE_PSLLI_W,
7752 INTRINS_SSE_PSRLI_D,
7753 INTRINS_SSE_PSRAI_D,
7754 INTRINS_SSE_PSLLI_D,
7755 INTRINS_SSE_PSRLI_Q,
7756 INTRINS_SSE_PSLLI_Q,
7757 INTRINS_SSE_SQRT_PD,
7758 INTRINS_SSE_SQRT_PS,
7759 INTRINS_SSE_RSQRT_PS,
7761 INTRINS_SSE_CVTTPD2DQ,
7762 INTRINS_SSE_CVTTPS2DQ,
7763 INTRINS_SSE_CVTDQ2PD,
7764 INTRINS_SSE_CVTDQ2PS,
7765 INTRINS_SSE_CVTPD2DQ,
7766 INTRINS_SSE_CVTPS2DQ,
7767 INTRINS_SSE_CVTPD2PS,
7768 INTRINS_SSE_CVTPS2PD,
7771 INTRINS_SSE_PACKSSWB,
7772 INTRINS_SSE_PACKUSWB,
7773 INTRINS_SSE_PACKSSDW,
7774 INTRINS_SSE_PACKUSDW,
7779 INTRINS_SSE_ADDSUBPS,
7784 INTRINS_SSE_ADDSUBPD,
7787 INTRINS_SSE_PADDUSW,
7788 INTRINS_SSE_PSUBUSW,
7794 INTRINS_SSE_PADDUSB,
7795 INTRINS_SSE_PSUBUSB,
7807 static IntrinsicDesc intrinsics[] = {
7808 {INTRINS_MEMSET, "llvm.memset.p0i8.i32"},
7809 {INTRINS_MEMCPY, "llvm.memcpy.p0i8.p0i8.i32"},
7810 {INTRINS_SADD_OVF_I32, "llvm.sadd.with.overflow.i32"},
7811 {INTRINS_UADD_OVF_I32, "llvm.uadd.with.overflow.i32"},
7812 {INTRINS_SSUB_OVF_I32, "llvm.ssub.with.overflow.i32"},
7813 {INTRINS_USUB_OVF_I32, "llvm.usub.with.overflow.i32"},
7814 {INTRINS_SMUL_OVF_I32, "llvm.smul.with.overflow.i32"},
7815 {INTRINS_UMUL_OVF_I32, "llvm.umul.with.overflow.i32"},
7816 {INTRINS_SADD_OVF_I64, "llvm.sadd.with.overflow.i64"},
7817 {INTRINS_UADD_OVF_I64, "llvm.uadd.with.overflow.i64"},
7818 {INTRINS_SSUB_OVF_I64, "llvm.ssub.with.overflow.i64"},
7819 {INTRINS_USUB_OVF_I64, "llvm.usub.with.overflow.i64"},
7820 {INTRINS_SMUL_OVF_I64, "llvm.smul.with.overflow.i64"},
7821 {INTRINS_UMUL_OVF_I64, "llvm.umul.with.overflow.i64"},
7822 {INTRINS_SIN, "llvm.sin.f64"},
7823 {INTRINS_COS, "llvm.cos.f64"},
7824 {INTRINS_SQRT, "llvm.sqrt.f64"},
7825 /* This isn't an intrinsic, instead llvm seems to special case it by name */
7826 {INTRINS_FABS, "fabs"},
7827 {INTRINS_EXPECT_I8, "llvm.expect.i8"},
7828 {INTRINS_EXPECT_I1, "llvm.expect.i1"},
7829 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7830 {INTRINS_SSE_PMOVMSKB, "llvm.x86.sse2.pmovmskb.128"},
7831 {INTRINS_SSE_PSRLI_W, "llvm.x86.sse2.psrli.w"},
7832 {INTRINS_SSE_PSRAI_W, "llvm.x86.sse2.psrai.w"},
7833 {INTRINS_SSE_PSLLI_W, "llvm.x86.sse2.pslli.w"},
7834 {INTRINS_SSE_PSRLI_D, "llvm.x86.sse2.psrli.d"},
7835 {INTRINS_SSE_PSRAI_D, "llvm.x86.sse2.psrai.d"},
7836 {INTRINS_SSE_PSLLI_D, "llvm.x86.sse2.pslli.d"},
7837 {INTRINS_SSE_PSRLI_Q, "llvm.x86.sse2.psrli.q"},
7838 {INTRINS_SSE_PSLLI_Q, "llvm.x86.sse2.pslli.q"},
7839 {INTRINS_SSE_SQRT_PD, "llvm.x86.sse2.sqrt.pd"},
7840 {INTRINS_SSE_SQRT_PS, "llvm.x86.sse.sqrt.ps"},
7841 {INTRINS_SSE_RSQRT_PS, "llvm.x86.sse.rsqrt.ps"},
7842 {INTRINS_SSE_RCP_PS, "llvm.x86.sse.rcp.ps"},
7843 {INTRINS_SSE_CVTTPD2DQ, "llvm.x86.sse2.cvttpd2dq"},
7844 {INTRINS_SSE_CVTTPS2DQ, "llvm.x86.sse2.cvttps2dq"},
7845 {INTRINS_SSE_CVTDQ2PD, "llvm.x86.sse2.cvtdq2pd"},
7846 {INTRINS_SSE_CVTDQ2PS, "llvm.x86.sse2.cvtdq2ps"},
7847 {INTRINS_SSE_CVTPD2DQ, "llvm.x86.sse2.cvtpd2dq"},
7848 {INTRINS_SSE_CVTPS2DQ, "llvm.x86.sse2.cvtps2dq"},
7849 {INTRINS_SSE_CVTPD2PS, "llvm.x86.sse2.cvtpd2ps"},
7850 {INTRINS_SSE_CVTPS2PD, "llvm.x86.sse2.cvtps2pd"},
7851 {INTRINS_SSE_CMPPD, "llvm.x86.sse2.cmp.pd"},
7852 {INTRINS_SSE_CMPPS, "llvm.x86.sse.cmp.ps"},
7853 {INTRINS_SSE_PACKSSWB, "llvm.x86.sse2.packsswb.128"},
7854 {INTRINS_SSE_PACKUSWB, "llvm.x86.sse2.packuswb.128"},
7855 {INTRINS_SSE_PACKSSDW, "llvm.x86.sse2.packssdw.128"},
7856 {INTRINS_SSE_PACKUSDW, "llvm.x86.sse41.packusdw"},
7857 {INTRINS_SSE_MINPS, "llvm.x86.sse.min.ps"},
7858 {INTRINS_SSE_MAXPS, "llvm.x86.sse.max.ps"},
7859 {INTRINS_SSE_HADDPS, "llvm.x86.sse3.hadd.ps"},
7860 {INTRINS_SSE_HSUBPS, "llvm.x86.sse3.hsub.ps"},
7861 {INTRINS_SSE_ADDSUBPS, "llvm.x86.sse3.addsub.ps"},
7862 {INTRINS_SSE_MINPD, "llvm.x86.sse2.min.pd"},
7863 {INTRINS_SSE_MAXPD, "llvm.x86.sse2.max.pd"},
7864 {INTRINS_SSE_HADDPD, "llvm.x86.sse3.hadd.pd"},
7865 {INTRINS_SSE_HSUBPD, "llvm.x86.sse3.hsub.pd"},
7866 {INTRINS_SSE_ADDSUBPD, "llvm.x86.sse3.addsub.pd"},
7867 {INTRINS_SSE_PADDSW, "llvm.x86.sse2.padds.w"},
7868 {INTRINS_SSE_PSUBSW, "llvm.x86.sse2.psubs.w"},
7869 {INTRINS_SSE_PADDUSW, "llvm.x86.sse2.paddus.w"},
7870 {INTRINS_SSE_PSUBUSW, "llvm.x86.sse2.psubus.w"},
7871 {INTRINS_SSE_PAVGW, "llvm.x86.sse2.pavg.w"},
7872 {INTRINS_SSE_PMULHW, "llvm.x86.sse2.pmulh.w"},
7873 {INTRINS_SSE_PMULHU, "llvm.x86.sse2.pmulhu.w"},
7874 {INTRINS_SE_PADDSB, "llvm.x86.sse2.padds.b"},
7875 {INTRINS_SSE_PSUBSB, "llvm.x86.sse2.psubs.b"},
7876 {INTRINS_SSE_PADDUSB, "llvm.x86.sse2.paddus.b"},
7877 {INTRINS_SSE_PSUBUSB, "llvm.x86.sse2.psubus.b"},
7878 {INTRINS_SSE_PAVGB, "llvm.x86.sse2.pavg.b"},
7879 {INTRINS_SSE_PAUSE, "llvm.x86.sse2.pause"}
7884 add_sse_binary (LLVMModuleRef module, const char *name, int type)
7886 LLVMTypeRef ret_type = type_to_simd_type (type);
7887 AddFunc2 (module, name, ret_type, ret_type, ret_type);
7891 add_intrinsic (LLVMModuleRef module, int id)
7894 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7895 LLVMTypeRef ret_type, arg_types [16];
7898 name = g_hash_table_lookup (intrins_id_to_name, GINT_TO_POINTER (id));
7902 case INTRINS_MEMSET: {
7903 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
7905 AddFunc (module, name, LLVMVoidType (), params, 5);
7908 case INTRINS_MEMCPY: {
7909 LLVMTypeRef params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
7911 AddFunc (module, name, LLVMVoidType (), params, 5);
7914 case INTRINS_SADD_OVF_I32:
7915 case INTRINS_UADD_OVF_I32:
7916 case INTRINS_SSUB_OVF_I32:
7917 case INTRINS_USUB_OVF_I32:
7918 case INTRINS_SMUL_OVF_I32:
7919 case INTRINS_UMUL_OVF_I32: {
7920 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
7921 LLVMTypeRef params [] = { LLVMInt32Type (), LLVMInt32Type () };
7922 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i32, 2, FALSE);
7924 AddFunc (module, name, ret_type, params, 2);
7927 case INTRINS_SADD_OVF_I64:
7928 case INTRINS_UADD_OVF_I64:
7929 case INTRINS_SSUB_OVF_I64:
7930 case INTRINS_USUB_OVF_I64:
7931 case INTRINS_SMUL_OVF_I64:
7932 case INTRINS_UMUL_OVF_I64: {
7933 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
7934 LLVMTypeRef params [] = { LLVMInt64Type (), LLVMInt64Type () };
7935 LLVMTypeRef ret_type = LLVMStructType (ovf_res_i64, 2, FALSE);
7937 AddFunc (module, name, ret_type, params, 2);
7943 case INTRINS_FABS: {
7944 LLVMTypeRef params [] = { LLVMDoubleType () };
7946 AddFunc (module, name, LLVMDoubleType (), params, 1);
7949 case INTRINS_EXPECT_I8:
7950 AddFunc2 (module, name, LLVMInt8Type (), LLVMInt8Type (), LLVMInt8Type ());
7952 case INTRINS_EXPECT_I1:
7953 AddFunc2 (module, name, LLVMInt1Type (), LLVMInt1Type (), LLVMInt1Type ());
7955 #if defined(TARGET_AMD64) || defined(TARGET_X86)
7956 case INTRINS_SSE_PMOVMSKB:
7958 ret_type = LLVMInt32Type ();
7959 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
7960 AddFunc (module, name, ret_type, arg_types, 1);
7962 case INTRINS_SSE_PSRLI_W:
7963 case INTRINS_SSE_PSRAI_W:
7964 case INTRINS_SSE_PSLLI_W:
7966 ret_type = type_to_simd_type (MONO_TYPE_I2);
7967 arg_types [0] = ret_type;
7968 arg_types [1] = LLVMInt32Type ();
7969 AddFunc (module, name, ret_type, arg_types, 2);
7971 case INTRINS_SSE_PSRLI_D:
7972 case INTRINS_SSE_PSRAI_D:
7973 case INTRINS_SSE_PSLLI_D:
7974 ret_type = type_to_simd_type (MONO_TYPE_I4);
7975 arg_types [0] = ret_type;
7976 arg_types [1] = LLVMInt32Type ();
7977 AddFunc (module, name, ret_type, arg_types, 2);
7979 case INTRINS_SSE_PSRLI_Q:
7980 case INTRINS_SSE_PSLLI_Q:
7981 ret_type = type_to_simd_type (MONO_TYPE_I8);
7982 arg_types [0] = ret_type;
7983 arg_types [1] = LLVMInt32Type ();
7984 AddFunc (module, name, ret_type, arg_types, 2);
7986 case INTRINS_SSE_SQRT_PD:
7988 ret_type = type_to_simd_type (MONO_TYPE_R8);
7989 arg_types [0] = ret_type;
7990 AddFunc (module, name, ret_type, arg_types, 1);
7992 case INTRINS_SSE_SQRT_PS:
7993 ret_type = type_to_simd_type (MONO_TYPE_R4);
7994 arg_types [0] = ret_type;
7995 AddFunc (module, name, ret_type, arg_types, 1);
7997 case INTRINS_SSE_RSQRT_PS:
7998 ret_type = type_to_simd_type (MONO_TYPE_R4);
7999 arg_types [0] = ret_type;
8000 AddFunc (module, name, ret_type, arg_types, 1);
8002 case INTRINS_SSE_RCP_PS:
8003 ret_type = type_to_simd_type (MONO_TYPE_R4);
8004 arg_types [0] = ret_type;
8005 AddFunc (module, name, ret_type, arg_types, 1);
8007 case INTRINS_SSE_CVTTPD2DQ:
8008 ret_type = type_to_simd_type (MONO_TYPE_I4);
8009 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8010 AddFunc (module, name, ret_type, arg_types, 1);
8012 case INTRINS_SSE_CVTTPS2DQ:
8013 ret_type = type_to_simd_type (MONO_TYPE_I4);
8014 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8015 AddFunc (module, name, ret_type, arg_types, 1);
8017 case INTRINS_SSE_CVTDQ2PD:
8018 /* Conversion ops */
8019 ret_type = type_to_simd_type (MONO_TYPE_R8);
8020 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8021 AddFunc (module, name, ret_type, arg_types, 1);
8023 case INTRINS_SSE_CVTDQ2PS:
8024 ret_type = type_to_simd_type (MONO_TYPE_R4);
8025 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8026 AddFunc (module, name, ret_type, arg_types, 1);
8028 case INTRINS_SSE_CVTPD2DQ:
8029 ret_type = type_to_simd_type (MONO_TYPE_I4);
8030 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8031 AddFunc (module, name, ret_type, arg_types, 1);
8033 case INTRINS_SSE_CVTPS2DQ:
8034 ret_type = type_to_simd_type (MONO_TYPE_I4);
8035 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8036 AddFunc (module, name, ret_type, arg_types, 1);
8038 case INTRINS_SSE_CVTPD2PS:
8039 ret_type = type_to_simd_type (MONO_TYPE_R4);
8040 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
8041 AddFunc (module, name, ret_type, arg_types, 1);
8043 case INTRINS_SSE_CVTPS2PD:
8044 ret_type = type_to_simd_type (MONO_TYPE_R8);
8045 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
8046 AddFunc (module, name, ret_type, arg_types, 1);
8048 case INTRINS_SSE_CMPPD:
8050 ret_type = type_to_simd_type (MONO_TYPE_R8);
8051 arg_types [0] = ret_type;
8052 arg_types [1] = ret_type;
8053 arg_types [2] = LLVMInt8Type ();
8054 AddFunc (module, name, ret_type, arg_types, 3);
8056 case INTRINS_SSE_CMPPS:
8057 ret_type = type_to_simd_type (MONO_TYPE_R4);
8058 arg_types [0] = ret_type;
8059 arg_types [1] = ret_type;
8060 arg_types [2] = LLVMInt8Type ();
8061 AddFunc (module, name, ret_type, arg_types, 3);
8063 case INTRINS_SSE_PACKSSWB:
8064 case INTRINS_SSE_PACKUSWB:
8065 case INTRINS_SSE_PACKSSDW:
8067 ret_type = type_to_simd_type (MONO_TYPE_I1);
8068 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
8069 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
8070 AddFunc (module, name, ret_type, arg_types, 2);
8072 case INTRINS_SSE_PACKUSDW:
8073 ret_type = type_to_simd_type (MONO_TYPE_I2);
8074 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
8075 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
8076 AddFunc (module, name, ret_type, arg_types, 2);
8078 /* SSE Binary ops */
8079 case INTRINS_SSE_PADDSW:
8080 case INTRINS_SSE_PSUBSW:
8081 case INTRINS_SSE_PADDUSW:
8082 case INTRINS_SSE_PSUBUSW:
8083 case INTRINS_SSE_PAVGW:
8084 case INTRINS_SSE_PMULHW:
8085 case INTRINS_SSE_PMULHU:
8086 add_sse_binary (module, name, MONO_TYPE_I2);
8088 case INTRINS_SSE_MINPS:
8089 case INTRINS_SSE_MAXPS:
8090 case INTRINS_SSE_HADDPS:
8091 case INTRINS_SSE_HSUBPS:
8092 case INTRINS_SSE_ADDSUBPS:
8093 add_sse_binary (module, name, MONO_TYPE_R4);
8095 case INTRINS_SSE_MINPD:
8096 case INTRINS_SSE_MAXPD:
8097 case INTRINS_SSE_HADDPD:
8098 case INTRINS_SSE_HSUBPD:
8099 case INTRINS_SSE_ADDSUBPD:
8100 add_sse_binary (module, name, MONO_TYPE_R8);
8102 case INTRINS_SE_PADDSB:
8103 case INTRINS_SSE_PSUBSB:
8104 case INTRINS_SSE_PADDUSB:
8105 case INTRINS_SSE_PSUBUSB:
8106 case INTRINS_SSE_PAVGB:
8107 add_sse_binary (module, name, MONO_TYPE_I1);
8109 case INTRINS_SSE_PAUSE:
8110 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
8114 g_assert_not_reached ();
8120 get_intrinsic (EmitContext *ctx, const char *name)
8122 #if LLVM_API_VERSION > 100
8126 * Every method is emitted into its own module so
8127 * we can add intrinsics on demand.
8129 res = LLVMGetNamedFunction (ctx->lmodule, name);
8133 /* No locking needed */
8134 id = GPOINTER_TO_INT (g_hash_table_lookup (intrins_name_to_id, name));
8137 printf ("%s\n", name);
8138 g_assert (id != -1);
8139 add_intrinsic (ctx->lmodule, id);
8140 res = LLVMGetNamedFunction (ctx->lmodule, name);
8148 res = LLVMGetNamedFunction (ctx->lmodule, name);
8155 add_intrinsics (LLVMModuleRef module)
8159 /* Emit declarations of instrinsics */
8161 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
8162 * type doesn't seem to do any locking.
8164 for (i = 0; i < INTRINS_NUM; ++i)
8165 add_intrinsic (module, i);
8169 AddFunc (module, "mono_personality", LLVMVoidType (), NULL, 0);
8171 AddFunc (module, "llvm_resume_unwind_trampoline", LLVMVoidType (), NULL, 0);
8174 /* Load/Store intrinsics */
8176 LLVMTypeRef arg_types [5];
8180 for (i = 1; i <= 8; i *= 2) {
8181 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
8182 arg_types [1] = LLVMInt32Type ();
8183 arg_types [2] = LLVMInt1Type ();
8184 arg_types [3] = LLVMInt32Type ();
8185 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
8186 AddFunc (module, name, LLVMIntType (i * 8), arg_types, 4);
8188 arg_types [0] = LLVMIntType (i * 8);
8189 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
8190 arg_types [2] = LLVMInt32Type ();
8191 arg_types [3] = LLVMInt1Type ();
8192 arg_types [4] = LLVMInt32Type ();
8193 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
8194 AddFunc (module, name, LLVMVoidType (), arg_types, 5);
8200 add_types (MonoLLVMModule *module)
8202 module->ptr_type = LLVMPointerType (sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
8206 mono_llvm_init (void)
8211 mono_native_tls_alloc (¤t_cfg_tls_id, NULL);
8213 h = g_hash_table_new (NULL, NULL);
8214 for (i = 0; i < INTRINS_NUM; ++i)
8215 g_hash_table_insert (h, GINT_TO_POINTER (intrinsics [i].id), (gpointer)intrinsics [i].name);
8216 intrins_id_to_name = h;
8218 h = g_hash_table_new (g_str_hash, g_str_equal);
8219 for (i = 0; i < INTRINS_NUM; ++i)
8220 g_hash_table_insert (h, (gpointer)intrinsics [i].name, GINT_TO_POINTER (intrinsics [i].id + 1));
8221 intrins_name_to_id = h;
8225 init_jit_module (MonoDomain *domain)
8227 MonoJitDomainInfo *dinfo;
8228 MonoLLVMModule *module;
8231 dinfo = domain_jit_info (domain);
8232 if (dinfo->llvm_module)
8235 mono_loader_lock ();
8237 if (dinfo->llvm_module) {
8238 mono_loader_unlock ();
8242 module = g_new0 (MonoLLVMModule, 1);
8244 name = g_strdup_printf ("mono-%s", domain->friendly_name);
8245 module->lmodule = LLVMModuleCreateWithName (name);
8246 module->context = LLVMGetGlobalContext ();
8248 module->mono_ee = (MonoEERef*)mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->lmodule), alloc_cb, emitted_cb, exception_cb, dlsym_cb, &module->ee);
8250 add_intrinsics (module->lmodule);
8253 module->llvm_types = g_hash_table_new (NULL, NULL);
8255 #if LLVM_API_VERSION < 100
8256 MonoJitICallInfo *info;
8258 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
8260 LLVMAddGlobalMapping (module->ee, LLVMGetNamedFunction (module->lmodule, "llvm_resume_unwind_trampoline"), (void*)info->func);
8263 mono_memory_barrier ();
8265 dinfo->llvm_module = module;
8267 mono_loader_unlock ();
8271 mono_llvm_cleanup (void)
8273 MonoLLVMModule *module = &aot_module;
8275 if (module->lmodule)
8276 LLVMDisposeModule (module->lmodule);
8278 if (module->context)
8279 LLVMContextDispose (module->context);
8283 mono_llvm_free_domain_info (MonoDomain *domain)
8285 MonoJitDomainInfo *info = domain_jit_info (domain);
8286 MonoLLVMModule *module = (MonoLLVMModule*)info->llvm_module;
8292 if (module->llvm_types)
8293 g_hash_table_destroy (module->llvm_types);
8295 mono_llvm_dispose_ee (module->mono_ee);
8297 if (module->bb_names) {
8298 for (i = 0; i < module->bb_names_len; ++i)
8299 g_free (module->bb_names [i]);
8300 g_free (module->bb_names);
8302 //LLVMDisposeModule (module->module);
8306 info->llvm_module = NULL;
8310 mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, gboolean emit_dwarf, gboolean static_link, gboolean llvm_only)
8312 MonoLLVMModule *module = &aot_module;
8314 /* Delete previous module */
8315 if (module->plt_entries)
8316 g_hash_table_destroy (module->plt_entries);
8317 if (module->lmodule)
8318 LLVMDisposeModule (module->lmodule);
8320 memset (module, 0, sizeof (aot_module));
8322 module->lmodule = LLVMModuleCreateWithName ("aot");
8323 module->assembly = assembly;
8324 module->global_prefix = g_strdup (global_prefix);
8325 module->got_symbol = g_strdup_printf ("%s_llvm_got", global_prefix);
8326 module->eh_frame_symbol = g_strdup_printf ("%s_eh_frame", global_prefix);
8327 module->get_method_symbol = g_strdup_printf ("%s_get_method", global_prefix);
8328 module->get_unbox_tramp_symbol = g_strdup_printf ("%s_get_unbox_tramp", global_prefix);
8329 module->external_symbols = TRUE;
8330 module->emit_dwarf = emit_dwarf;
8331 module->static_link = static_link;
8332 module->llvm_only = llvm_only;
8333 /* The first few entries are reserved */
8334 module->max_got_offset = 16;
8335 module->context = LLVMContextCreate ();
8338 /* clang ignores our debug info because it has an invalid version */
8339 module->emit_dwarf = FALSE;
8341 add_intrinsics (module->lmodule);
8344 #if LLVM_API_VERSION > 100
8345 if (module->emit_dwarf) {
8346 char *dir, *build_info, *s, *cu_name;
8348 module->di_builder = mono_llvm_create_di_builder (module->lmodule);
8351 dir = g_strdup (".");
8352 build_info = mono_get_runtime_build_info ();
8353 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
8354 cu_name = g_path_get_basename (assembly->image->name);
8355 module->cu = mono_llvm_di_create_compile_unit (module->di_builder, cu_name, dir, s);
8357 g_free (build_info);
8364 * We couldn't compute the type of the LLVM global representing the got because
8365 * its size is only known after all the methods have been emitted. So create
8366 * a dummy variable, and replace all uses it with the real got variable when
8367 * its size is known in mono_llvm_emit_aot_module ().
8370 LLVMTypeRef got_type = LLVMArrayType (module->ptr_type, 0);
8372 module->got_var = LLVMAddGlobal (module->lmodule, got_type, "mono_dummy_got");
8373 LLVMSetInitializer (module->got_var, LLVMConstNull (got_type));
8376 /* Add initialization array */
8378 LLVMTypeRef inited_type = LLVMArrayType (LLVMInt8Type (), 0);
8380 module->inited_var = LLVMAddGlobal (aot_module.lmodule, inited_type, "mono_inited_tmp");
8381 LLVMSetInitializer (module->inited_var, LLVMConstNull (inited_type));
8385 emit_init_icall_wrappers (module);
8387 emit_llvm_code_start (module);
8389 /* Add a dummy personality function */
8390 if (!use_debug_personality) {
8391 LLVMValueRef personality = LLVMAddFunction (module->lmodule, default_personality_name, LLVMFunctionType (LLVMInt32Type (), NULL, 0, TRUE));
8392 LLVMSetLinkage (personality, LLVMExternalLinkage);
8393 mark_as_used (module, personality);
8396 /* Add a reference to the c++ exception we throw/catch */
8398 LLVMTypeRef exc = LLVMPointerType (LLVMInt8Type (), 0);
8399 module->sentinel_exception = LLVMAddGlobal (module->lmodule, exc, "_ZTIPi");
8400 LLVMSetLinkage (module->sentinel_exception, LLVMExternalLinkage);
8401 mono_llvm_set_is_constant (module->sentinel_exception);
8404 module->llvm_types = g_hash_table_new (NULL, NULL);
8405 module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
8406 module->plt_entries_ji = g_hash_table_new (NULL, NULL);
8407 module->direct_callables = g_hash_table_new (g_str_hash, g_str_equal);
8408 module->method_to_lmethod = g_hash_table_new (NULL, NULL);
8409 module->idx_to_lmethod = g_hash_table_new (NULL, NULL);
8410 module->idx_to_unbox_tramp = g_hash_table_new (NULL, NULL);
8411 module->method_to_callers = g_hash_table_new (NULL, NULL);
8415 llvm_array_from_uints (LLVMTypeRef el_type, guint32 *values, int nvalues)
8418 LLVMValueRef res, *vals;
8420 vals = g_new0 (LLVMValueRef, nvalues);
8421 for (i = 0; i < nvalues; ++i)
8422 vals [i] = LLVMConstInt (LLVMInt32Type (), values [i], FALSE);
8423 res = LLVMConstArray (LLVMInt32Type (), vals, nvalues);
8429 llvm_array_from_bytes (guint8 *values, int nvalues)
8432 LLVMValueRef res, *vals;
8434 vals = g_new0 (LLVMValueRef, nvalues);
8435 for (i = 0; i < nvalues; ++i)
8436 vals [i] = LLVMConstInt (LLVMInt8Type (), values [i], FALSE);
8437 res = LLVMConstArray (LLVMInt8Type (), vals, nvalues);
8442 * mono_llvm_emit_aot_file_info:
8444 * Emit the MonoAotFileInfo structure.
8445 * Same as emit_aot_file_info () in aot-compiler.c.
8448 mono_llvm_emit_aot_file_info (MonoAotFileInfo *info, gboolean has_jitted_code)
8450 MonoLLVMModule *module = &aot_module;
8452 /* Save these for later */
8453 memcpy (&module->aot_info, info, sizeof (MonoAotFileInfo));
8454 module->has_jitted_code = has_jitted_code;
8458 * mono_llvm_emit_aot_data:
8460 * Emit the binary data DATA pointed to by symbol SYMBOL.
8463 mono_llvm_emit_aot_data (const char *symbol, guint8 *data, int data_len)
8465 MonoLLVMModule *module = &aot_module;
8469 type = LLVMArrayType (LLVMInt8Type (), data_len);
8470 d = LLVMAddGlobal (module->lmodule, type, symbol);
8471 LLVMSetVisibility (d, LLVMHiddenVisibility);
8472 LLVMSetLinkage (d, LLVMInternalLinkage);
8473 LLVMSetInitializer (d, mono_llvm_create_constant_data_array (data, data_len));
8474 mono_llvm_set_is_constant (d);
8477 /* Add a reference to a global defined in JITted code */
8479 AddJitGlobal (MonoLLVMModule *module, LLVMTypeRef type, const char *name)
8484 s = g_strdup_printf ("%s%s", module->global_prefix, name);
8485 v = LLVMAddGlobal (module->lmodule, LLVMInt8Type (), s);
8491 emit_aot_file_info (MonoLLVMModule *module)
8493 LLVMTypeRef file_info_type;
8494 LLVMTypeRef *eltypes, eltype;
8495 LLVMValueRef info_var;
8496 LLVMValueRef *fields;
8497 int i, nfields, tindex;
8498 MonoAotFileInfo *info;
8499 LLVMModuleRef lmodule = module->lmodule;
8501 info = &module->aot_info;
8503 /* Create an LLVM type to represent MonoAotFileInfo */
8504 nfields = 2 + MONO_AOT_FILE_INFO_NUM_SYMBOLS + 16 + 5;
8505 eltypes = g_new (LLVMTypeRef, nfields);
8507 eltypes [tindex ++] = LLVMInt32Type ();
8508 eltypes [tindex ++] = LLVMInt32Type ();
8510 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
8511 eltypes [tindex ++] = LLVMPointerType (LLVMInt8Type (), 0);
8513 for (i = 0; i < 15; ++i)
8514 eltypes [tindex ++] = LLVMInt32Type ();
8516 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TABLE_NUM);
8517 for (i = 0; i < 4; ++i)
8518 eltypes [tindex ++] = LLVMArrayType (LLVMInt32Type (), MONO_AOT_TRAMP_NUM);
8519 eltypes [tindex ++] = LLVMArrayType (LLVMInt8Type (), 16);
8520 g_assert (tindex == nfields);
8521 file_info_type = LLVMStructCreateNamed (module->context, "MonoAotFileInfo");
8522 LLVMStructSetBody (file_info_type, eltypes, nfields, FALSE);
8524 info_var = LLVMAddGlobal (lmodule, file_info_type, "mono_aot_file_info");
8525 if (module->static_link) {
8526 LLVMSetVisibility (info_var, LLVMHiddenVisibility);
8527 LLVMSetLinkage (info_var, LLVMInternalLinkage);
8529 fields = g_new (LLVMValueRef, nfields);
8531 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->version, FALSE);
8532 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->dummy, FALSE);
8536 * We use LLVMGetNamedGlobal () for symbol which are defined in LLVM code, and LLVMAddGlobal ()
8537 * for symbols defined in the .s file emitted by the aot compiler.
8539 eltype = eltypes [tindex];
8540 if (module->llvm_only)
8541 fields [tindex ++] = LLVMConstNull (eltype);
8543 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_got");
8544 fields [tindex ++] = module->got_var;
8545 /* llc defines this directly */
8546 if (!module->llvm_only) {
8547 fields [tindex ++] = LLVMAddGlobal (lmodule, eltype, module->eh_frame_symbol);
8548 fields [tindex ++] = LLVMConstNull (eltype);
8549 fields [tindex ++] = LLVMConstNull (eltype);
8551 fields [tindex ++] = LLVMConstNull (eltype);
8552 fields [tindex ++] = module->get_method;
8553 fields [tindex ++] = module->get_unbox_tramp;
8555 if (module->has_jitted_code) {
8556 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_start");
8557 fields [tindex ++] = AddJitGlobal (module, eltype, "jit_code_end");
8559 fields [tindex ++] = LLVMConstNull (eltype);
8560 fields [tindex ++] = LLVMConstNull (eltype);
8562 if (!module->llvm_only)
8563 fields [tindex ++] = AddJitGlobal (module, eltype, "method_addresses");
8565 fields [tindex ++] = LLVMConstNull (eltype);
8566 if (info->flags & MONO_AOT_FILE_FLAG_SEPARATE_DATA) {
8567 for (i = 0; i < MONO_AOT_TABLE_NUM; ++i)
8568 fields [tindex ++] = LLVMConstNull (eltype);
8570 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "blob");
8571 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_name_table");
8572 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "class_info_offsets");
8573 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "method_info_offsets");
8574 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "ex_info_offsets");
8575 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_info_offsets");
8576 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "extra_method_table");
8577 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "got_info_offsets");
8578 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "llvm_got_info_offsets");
8579 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "image_table");
8581 /* Not needed (mem_end) */
8582 fields [tindex ++] = LLVMConstNull (eltype);
8583 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_guid");
8584 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "runtime_version");
8585 if (info->trampoline_size [0]) {
8586 fields [tindex ++] = AddJitGlobal (module, eltype, "specific_trampolines");
8587 fields [tindex ++] = AddJitGlobal (module, eltype, "static_rgctx_trampolines");
8588 fields [tindex ++] = AddJitGlobal (module, eltype, "imt_thunks");
8589 fields [tindex ++] = AddJitGlobal (module, eltype, "gsharedvt_arg_trampolines");
8591 fields [tindex ++] = LLVMConstNull (eltype);
8592 fields [tindex ++] = LLVMConstNull (eltype);
8593 fields [tindex ++] = LLVMConstNull (eltype);
8594 fields [tindex ++] = LLVMConstNull (eltype);
8596 if (module->static_link && !module->llvm_only)
8597 fields [tindex ++] = AddJitGlobal (module, eltype, "globals");
8599 fields [tindex ++] = LLVMConstNull (eltype);
8600 fields [tindex ++] = LLVMGetNamedGlobal (lmodule, "assembly_name");
8601 if (!module->llvm_only) {
8602 fields [tindex ++] = AddJitGlobal (module, eltype, "plt");
8603 fields [tindex ++] = AddJitGlobal (module, eltype, "plt_end");
8604 fields [tindex ++] = AddJitGlobal (module, eltype, "unwind_info");
8605 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines");
8606 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampolines_end");
8607 fields [tindex ++] = AddJitGlobal (module, eltype, "unbox_trampoline_addresses");
8609 fields [tindex ++] = LLVMConstNull (eltype);
8610 fields [tindex ++] = LLVMConstNull (eltype);
8611 fields [tindex ++] = LLVMConstNull (eltype);
8612 fields [tindex ++] = LLVMConstNull (eltype);
8613 fields [tindex ++] = LLVMConstNull (eltype);
8614 fields [tindex ++] = LLVMConstNull (eltype);
8617 for (i = 0; i < MONO_AOT_FILE_INFO_NUM_SYMBOLS; ++i)
8618 fields [2 + i] = LLVMConstBitCast (fields [2 + i], eltype);
8621 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_got_offset_base, FALSE);
8622 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->got_size, FALSE);
8623 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->plt_size, FALSE);
8624 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nmethods, FALSE);
8625 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->flags, FALSE);
8626 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->opts, FALSE);
8627 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->simd_opts, FALSE);
8628 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->gc_name_index, FALSE);
8629 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->num_rgctx_fetch_trampolines, FALSE);
8630 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->double_align, FALSE);
8631 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->long_align, FALSE);
8632 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->generic_tramp_num, FALSE);
8633 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->tramp_page_size, FALSE);
8634 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->nshared_got_entries, FALSE);
8635 fields [tindex ++] = LLVMConstInt (LLVMInt32Type (), info->datafile_size, FALSE);
8637 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->table_offsets, MONO_AOT_TABLE_NUM);
8638 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->num_trampolines, MONO_AOT_TRAMP_NUM);
8639 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_got_offset_base, MONO_AOT_TRAMP_NUM);
8640 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->trampoline_size, MONO_AOT_TRAMP_NUM);
8641 fields [tindex ++] = llvm_array_from_uints (LLVMInt32Type (), info->tramp_page_code_offsets, MONO_AOT_TRAMP_NUM);
8643 fields [tindex ++] = llvm_array_from_bytes (info->aotid, 16);
8644 g_assert (tindex == nfields);
8646 LLVMSetInitializer (info_var, LLVMConstNamedStruct (file_info_type, fields, nfields));
8648 if (module->static_link) {
8652 s = g_strdup_printf ("mono_aot_module_%s_info", module->assembly->aname.name);
8653 /* Get rid of characters which cannot occur in symbols */
8655 for (p = s; *p; ++p) {
8656 if (!(isalnum (*p) || *p == '_'))
8659 var = LLVMAddGlobal (module->lmodule, LLVMPointerType (LLVMInt8Type (), 0), s);
8661 LLVMSetInitializer (var, LLVMConstBitCast (LLVMGetNamedGlobal (module->lmodule, "mono_aot_file_info"), LLVMPointerType (LLVMInt8Type (), 0)));
8662 LLVMSetLinkage (var, LLVMExternalLinkage);
8667 * Emit the aot module into the LLVM bitcode file FILENAME.
8670 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
8672 LLVMTypeRef got_type, inited_type;
8673 LLVMValueRef real_got, real_inited;
8674 MonoLLVMModule *module = &aot_module;
8676 emit_llvm_code_end (module);
8679 * Create the real got variable and replace all uses of the dummy variable with
8682 got_type = LLVMArrayType (module->ptr_type, module->max_got_offset + 1);
8683 real_got = LLVMAddGlobal (module->lmodule, got_type, module->got_symbol);
8684 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
8685 if (module->external_symbols) {
8686 LLVMSetLinkage (real_got, LLVMExternalLinkage);
8687 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
8689 LLVMSetLinkage (real_got, LLVMInternalLinkage);
8691 mono_llvm_replace_uses_of (module->got_var, real_got);
8693 mark_as_used (&aot_module, real_got);
8695 /* Delete the dummy got so it doesn't become a global */
8696 LLVMDeleteGlobal (module->got_var);
8697 module->got_var = real_got;
8700 * Same for the init_var
8702 if (module->llvm_only) {
8703 inited_type = LLVMArrayType (LLVMInt8Type (), module->max_inited_idx + 1);
8704 real_inited = LLVMAddGlobal (module->lmodule, inited_type, "mono_inited");
8705 LLVMSetInitializer (real_inited, LLVMConstNull (inited_type));
8706 LLVMSetLinkage (real_inited, LLVMInternalLinkage);
8707 mono_llvm_replace_uses_of (module->inited_var, real_inited);
8708 LLVMDeleteGlobal (module->inited_var);
8711 if (module->llvm_only) {
8712 emit_get_method (&aot_module);
8713 emit_get_unbox_tramp (&aot_module);
8716 emit_llvm_used (&aot_module);
8717 emit_dbg_info (&aot_module, filename, cu_name);
8718 emit_aot_file_info (&aot_module);
8721 * Replace GOT entries for directly callable methods with the methods themselves.
8722 * It would be easier to implement this by predefining all methods before compiling
8723 * their bodies, but that couldn't handle the case when a method fails to compile
8726 if (module->llvm_only) {
8727 GHashTableIter iter;
8729 GSList *callers, *l;
8731 g_hash_table_iter_init (&iter, module->method_to_callers);
8732 while (g_hash_table_iter_next (&iter, (void**)&method, (void**)&callers)) {
8733 LLVMValueRef lmethod;
8735 if (method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)
8738 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, method);
8740 for (l = callers; l; l = l->next) {
8741 LLVMValueRef caller = (LLVMValueRef)l->data;
8743 mono_llvm_replace_uses_of (caller, lmethod);
8749 /* Replace PLT entries for directly callable methods with the methods themselves */
8751 GHashTableIter iter;
8753 LLVMValueRef callee;
8755 g_hash_table_iter_init (&iter, module->plt_entries_ji);
8756 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
8757 if (mono_aot_is_direct_callable (ji)) {
8758 LLVMValueRef lmethod;
8760 lmethod = (LLVMValueRef)g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
8761 /* The types might not match because the caller might pass an rgctx */
8762 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
8763 mono_llvm_replace_uses_of (callee, lmethod);
8764 mono_aot_mark_unused_llvm_plt_entry (ji);
8774 if (LLVMVerifyModule (module->lmodule, LLVMReturnStatusAction, &verifier_err)) {
8775 printf ("%s\n", verifier_err);
8776 g_assert_not_reached ();
8781 LLVMWriteBitcodeToFile (module->lmodule, filename);
8786 md_string (const char *s)
8788 return LLVMMDString (s, strlen (s));
8791 /* Debugging support */
8794 emit_dbg_info (MonoLLVMModule *module, const char *filename, const char *cu_name)
8796 LLVMModuleRef lmodule = module->lmodule;
8797 LLVMValueRef args [16], ver;
8800 * This can only be enabled when LLVM code is emitted into a separate object
8801 * file, since the AOT compiler also emits dwarf info,
8802 * and the abbrev indexes will not be correct since llvm has added its own
8805 if (!module->emit_dwarf)
8808 #if LLVM_API_VERSION > 100
8809 mono_llvm_di_builder_finalize (module->di_builder);
8811 LLVMValueRef cu_args [16], cu;
8813 char *build_info, *s, *dir;
8816 * Emit dwarf info in the form of LLVM metadata. There is some
8817 * out-of-date documentation at:
8818 * http://llvm.org/docs/SourceLevelDebugging.html
8819 * but most of this was gathered from the llvm and
8824 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_TAG_compile_unit, FALSE);
8825 /* CU name/compilation dir */
8826 dir = g_path_get_dirname (filename);
8827 args [0] = LLVMMDString (cu_name, strlen (cu_name));
8828 args [1] = LLVMMDString (dir, strlen (dir));
8829 cu_args [n_cuargs ++] = LLVMMDNode (args, 2);
8832 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_LANG_C99, FALSE);
8834 build_info = mono_get_runtime_build_info ();
8835 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
8836 cu_args [n_cuargs ++] = LLVMMDString (s, strlen (s));
8837 g_free (build_info);
8839 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8841 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
8842 /* Runtime version */
8843 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8845 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8846 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8848 if (module->subprogram_mds) {
8852 mds = g_new0 (LLVMValueRef, module->subprogram_mds->len);
8853 for (i = 0; i < module->subprogram_mds->len; ++i)
8854 mds [i] = (LLVMValueRef)g_ptr_array_index (module->subprogram_mds, i);
8855 cu_args [n_cuargs ++] = LLVMMDNode (mds, module->subprogram_mds->len);
8857 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8860 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8861 /* Imported modules */
8862 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
8864 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
8865 /* DebugEmissionKind = FullDebug */
8866 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8867 cu = LLVMMDNode (cu_args, n_cuargs);
8868 LLVMAddNamedMetadataOperand (lmodule, "llvm.dbg.cu", cu);
8871 #if LLVM_API_VERSION > 100
8872 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8873 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
8874 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8875 ver = LLVMMDNode (args, 3);
8876 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8878 args [0] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8879 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
8880 args [2] = LLVMConstInt (LLVMInt64Type (), 3, FALSE);
8881 ver = LLVMMDNode (args, 3);
8882 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8884 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8885 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
8886 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
8887 ver = LLVMMDNode (args, 3);
8888 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8890 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8891 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
8892 args [2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8893 ver = LLVMMDNode (args, 3);
8894 LLVMAddNamedMetadataOperand (lmodule, "llvm.module.flags", ver);
8899 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
8901 MonoLLVMModule *module = ctx->module;
8902 MonoDebugMethodInfo *minfo = ctx->minfo;
8903 char *source_file, *dir, *filename;
8904 LLVMValueRef md, args [16], ctx_args [16], md_args [64], type_args [16], ctx_md, type_md;
8905 MonoSymSeqPoint *sym_seq_points;
8911 mono_debug_symfile_get_seq_points (minfo, &source_file, NULL, NULL, &sym_seq_points, &n_seq_points);
8913 source_file = g_strdup ("<unknown>");
8914 dir = g_path_get_dirname (source_file);
8915 filename = g_path_get_basename (source_file);
8917 #if LLVM_API_VERSION > 100
8918 return mono_llvm_di_create_function (module->di_builder, module->cu, method, cfg->method->name, name, dir, filename, n_seq_points ? sym_seq_points [0].line : 1);
8921 ctx_args [0] = LLVMConstInt (LLVMInt32Type (), 0x29, FALSE);
8922 args [0] = md_string (filename);
8923 args [1] = md_string (dir);
8924 ctx_args [1] = LLVMMDNode (args, 2);
8925 ctx_md = LLVMMDNode (ctx_args, 2);
8927 type_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subroutine_type, FALSE);
8928 type_args [1] = NULL;
8929 type_args [2] = NULL;
8930 type_args [3] = LLVMMDString ("", 0);
8931 type_args [4] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8932 type_args [5] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8933 type_args [6] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8934 type_args [7] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
8935 type_args [8] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8936 type_args [9] = NULL;
8937 type_args [10] = NULL;
8938 type_args [11] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8939 type_args [12] = NULL;
8940 type_args [13] = NULL;
8941 type_args [14] = NULL;
8942 type_md = LLVMMDNode (type_args, 14);
8944 /* http://llvm.org/docs/SourceLevelDebugging.html#subprogram-descriptors */
8945 md_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subprogram, FALSE);
8946 /* Source directory + file pair */
8947 args [0] = md_string (filename);
8948 args [1] = md_string (dir);
8949 md_args [1] = LLVMMDNode (args ,2);
8950 md_args [2] = ctx_md;
8951 md_args [3] = md_string (cfg->method->name);
8952 md_args [4] = md_string (name);
8953 md_args [5] = md_string (name);
8956 md_args [6] = LLVMConstInt (LLVMInt32Type (), sym_seq_points [0].line, FALSE);
8958 md_args [6] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8960 md_args [7] = type_md;
8962 md_args [8] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8964 md_args [9] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8966 md_args [10] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
8967 /* Index into a virtual function */
8968 md_args [11] = NULL;
8969 md_args [12] = NULL;
8971 md_args [13] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
8973 md_args [14] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
8974 /* Pointer to LLVM function */
8975 md_args [15] = method;
8976 /* Function template parameter */
8977 md_args [16] = NULL;
8978 /* Function declaration descriptor */
8979 md_args [17] = NULL;
8980 /* List of function variables */
8981 md_args [18] = LLVMMDNode (args, 0);
8983 md_args [19] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
8984 md = LLVMMDNode (md_args, 20);
8986 if (!module->subprogram_mds)
8987 module->subprogram_mds = g_ptr_array_new ();
8988 g_ptr_array_add (module->subprogram_mds, md);
8992 g_free (source_file);
8993 g_free (sym_seq_points);
8999 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
9001 MonoCompile *cfg = ctx->cfg;
9003 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
9004 MonoDebugSourceLocation *loc;
9005 LLVMValueRef loc_md;
9007 loc = mono_debug_symfile_lookup_location (ctx->minfo, cil_code - cfg->header->code);
9010 #if LLVM_API_VERSION > 100
9011 loc_md = mono_llvm_di_create_location (ctx->module->di_builder, ctx->dbg_md, loc->row, loc->column);
9012 mono_llvm_di_set_location (builder, loc_md);
9014 LLVMValueRef md_args [16];
9018 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->row, FALSE);
9019 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->column, FALSE);
9020 md_args [nmd_args ++] = ctx->dbg_md;
9021 md_args [nmd_args ++] = NULL;
9022 loc_md = LLVMMDNode (md_args, nmd_args);
9023 LLVMSetCurrentDebugLocation (builder, loc_md);
9025 mono_debug_symfile_free_location (loc);
9031 default_mono_llvm_unhandled_exception (void)
9033 MonoJitTlsData *jit_tls = mono_get_jit_tls ();
9034 MonoObject *target = mono_gchandle_get_target (jit_tls->thrown_exc);
9036 mono_unhandled_exception (target);
9037 mono_invoke_unhandled_exception_hook (target);
9038 g_assert_not_reached ();
9043 - Emit LLVM IR from the mono IR using the LLVM C API.
9044 - The original arch specific code remains, so we can fall back to it if we run
9045 into something we can't handle.
9049 A partial list of issues:
9050 - Handling of opcodes which can throw exceptions.
9052 In the mono JIT, these are implemented using code like this:
9059 push throw_pos - method
9060 call <exception trampoline>
9062 The problematic part is push throw_pos - method, which cannot be represented
9063 in the LLVM IR, since it does not support label values.
9064 -> this can be implemented in AOT mode using inline asm + labels, but cannot
9065 be implemented in JIT mode ?
9066 -> a possible but slower implementation would use the normal exception
9067 throwing code but it would need to control the placement of the throw code
9068 (it needs to be exactly after the compare+branch).
9069 -> perhaps add a PC offset intrinsics ?
9071 - efficient implementation of .ovf opcodes.
9073 These are currently implemented as:
9074 <ins which sets the condition codes>
9077 Some overflow opcodes are now supported by LLVM SVN.
9079 - exception handling, unwinding.
9080 - SSA is disabled for methods with exception handlers
9081 - How to obtain unwind info for LLVM compiled methods ?
9082 -> this is now solved by converting the unwind info generated by LLVM
9084 - LLVM uses the c++ exception handling framework, while we use our home grown
9085 code, and couldn't use the c++ one:
9086 - its not supported under VC++, other exotic platforms.
9087 - it might be impossible to support filter clauses with it.
9091 The trampolines need a predictable call sequence, since they need to disasm
9092 the calling code to obtain register numbers / offsets.
9094 LLVM currently generates this code in non-JIT mode:
9095 mov -0x98(%rax),%eax
9097 Here, the vtable pointer is lost.
9098 -> solution: use one vtable trampoline per class.
9100 - passing/receiving the IMT pointer/RGCTX.
9101 -> solution: pass them as normal arguments ?
9105 LLVM does not allow the specification of argument registers etc. This means
9106 that all calls are made according to the platform ABI.
9108 - passing/receiving vtypes.
9110 Vtypes passed/received in registers are handled by the front end by using
9111 a signature with scalar arguments, and loading the parts of the vtype into those
9114 Vtypes passed on the stack are handled using the 'byval' attribute.
9118 Supported though alloca, we need to emit the load/store code.
9122 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
9123 typed registers, so we have to keep track of the precise LLVM type of each vreg.
9124 This is made easier because the IR is already in SSA form.
9125 An additional problem is that our IR is not consistent with types, i.e. i32/ia64
9126 types are frequently used incorrectly.
9131 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then link
9132 it with the file containing the methods emitted by the JIT and the AOT data
9136 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
9137 * - each bblock should end with a branch
9138 * - setting the return value, making cfg->ret non-volatile
9139 * - avoid some transformations in the JIT which make it harder for us to generate
9141 * - use pointer types to help optimizations.