2 * mini-llvm.c: llvm "Backend" for the mono JIT
4 * Copyright 2009-2011 Novell Inc (http://www.novell.com)
5 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
9 #include <mono/metadata/debug-helpers.h>
10 #include <mono/metadata/debug-mono-symfile.h>
11 #include <mono/metadata/mempool-internals.h>
12 #include <mono/utils/mono-tls.h>
13 #include <mono/utils/mono-dl.h>
14 #include <mono/utils/mono-time.h>
15 #include <mono/utils/freebsd-dwarf.h>
17 #ifndef __STDC_LIMIT_MACROS
18 #define __STDC_LIMIT_MACROS
20 #ifndef __STDC_CONSTANT_MACROS
21 #define __STDC_CONSTANT_MACROS
24 #include "llvm-c/Core.h"
25 #include "llvm-c/ExecutionEngine.h"
26 #include "llvm-c/BitWriter.h"
27 #include "llvm-c/Analysis.h"
29 #include "mini-llvm-cpp.h"
34 extern void *memset(void *, int, size_t);
35 void bzero (void *to, size_t count) { memset (to, 0, count); }
40 * Information associated by mono with LLVM modules.
44 LLVMValueRef throw, rethrow, throw_corlib_exception;
45 GHashTable *llvm_types;
47 const char *got_symbol;
48 GHashTable *plt_entries;
49 GHashTable *plt_entries_ji;
50 GHashTable *method_to_lmethod;
55 GPtrArray *subprogram_mds;
57 LLVMExecutionEngineRef ee;
58 gboolean external_symbols;
64 * Information associated by the backend with mono basic blocks.
67 LLVMBasicBlockRef bblock, end_bblock;
68 LLVMValueRef finally_ind;
69 gboolean added, invoke_target;
71 * If this bblock is the start of a finally clause, this is a list of bblocks it
72 * needs to branch to in ENDFINALLY.
74 GSList *call_handler_return_bbs;
76 * If this bblock is the start of a finally clause, this is the bblock that
77 * CALL_HANDLER needs to branch to.
79 LLVMBasicBlockRef call_handler_target_bb;
80 /* The list of switch statements generated by ENDFINALLY instructions */
81 GSList *endfinally_switch_ins_list;
86 * Structure containing emit state
91 /* Maps method names to the corresponding LLVMValueRef */
92 GHashTable *emitted_method_decls;
96 MonoLLVMModule *lmodule;
99 int sindex, default_index, ex_index;
100 LLVMBuilderRef builder;
101 LLVMValueRef *values, *addresses;
102 MonoType **vreg_cli_types;
104 MonoMethodSignature *sig;
106 GHashTable *region_to_handler;
107 LLVMBuilderRef alloca_builder;
108 LLVMValueRef last_alloca;
109 LLVMValueRef rgctx_arg;
110 LLVMTypeRef *vreg_types;
112 gboolean *unreachable;
114 LLVMValueRef imt_rgctx_loc;
115 GHashTable *llvm_types;
117 MonoDebugMethodInfo *minfo;
124 MonoBasicBlock *in_bb;
129 * Instruction metadata
130 * This is the same as ins_info, but LREG != IREG.
138 #define MINI_OP(a,b,dest,src1,src2) dest, src1, src2, ' ',
139 #define MINI_OP3(a,b,dest,src1,src2,src3) dest, src1, src2, src3,
146 /* keep in sync with the enum in mini.h */
149 #include "mini-ops.h"
154 #if SIZEOF_VOID_P == 4
155 #define GET_LONG_IMM(ins) (((guint64)(ins)->inst_ms_word << 32) | (guint64)(guint32)(ins)->inst_ls_word)
157 #define GET_LONG_IMM(ins) ((ins)->inst_imm)
160 #define LLVM_INS_INFO(opcode) (&llvm_ins_info [((opcode) - OP_START - 1) * 4])
163 #define TRACE_FAILURE(msg) do { printf ("%s\n", msg); } while (0)
165 #define TRACE_FAILURE(msg)
169 #define IS_TARGET_X86 1
171 #define IS_TARGET_X86 0
175 #define IS_TARGET_AMD64 1
177 #define IS_TARGET_AMD64 0
180 #define LLVM_FAILURE(ctx, reason) do { \
181 TRACE_FAILURE (reason); \
182 (ctx)->cfg->exception_message = g_strdup (reason); \
183 (ctx)->cfg->disable_llvm = TRUE; \
187 #define CHECK_FAILURE(ctx) do { \
188 if ((ctx)->cfg->disable_llvm) \
192 static LLVMIntPredicate cond_to_llvm_cond [] = {
205 static LLVMRealPredicate fpcond_to_llvm_cond [] = {
218 static MonoNativeTlsKey current_cfg_tls_id;
220 static MonoLLVMModule aot_module;
221 static int memset_param_count, memcpy_param_count;
222 static const char *memset_func_name;
223 static const char *memcpy_func_name;
225 static void init_jit_module (MonoDomain *domain);
227 static void emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code);
228 static LLVMValueRef emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name);
229 static void emit_dbg_info (MonoLLVMModule *lmodule, const char *filename, const char *cu_name);
234 * The LLVM type with width == sizeof (gpointer)
239 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
245 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
251 return sizeof (gpointer) == 8 ? LLVMPointerType (LLVMInt64Type (), 0) : LLVMPointerType (LLVMInt32Type (), 0);
257 * Return the size of the LLVM representation of the vtype T.
260 get_vtype_size (MonoType *t)
264 size = mono_class_value_size (mono_class_from_mono_type (t), NULL);
266 while (size < 2 * sizeof (gpointer) && mono_is_power_of_two (size) == -1)
273 * simd_class_to_llvm_type:
275 * Return the LLVM type corresponding to the Mono.SIMD class KLASS
278 simd_class_to_llvm_type (EmitContext *ctx, MonoClass *klass)
280 if (!strcmp (klass->name, "Vector2d")) {
281 return LLVMVectorType (LLVMDoubleType (), 2);
282 } else if (!strcmp (klass->name, "Vector2l")) {
283 return LLVMVectorType (LLVMInt64Type (), 2);
284 } else if (!strcmp (klass->name, "Vector2ul")) {
285 return LLVMVectorType (LLVMInt64Type (), 2);
286 } else if (!strcmp (klass->name, "Vector4i")) {
287 return LLVMVectorType (LLVMInt32Type (), 4);
288 } else if (!strcmp (klass->name, "Vector4ui")) {
289 return LLVMVectorType (LLVMInt32Type (), 4);
290 } else if (!strcmp (klass->name, "Vector4f")) {
291 return LLVMVectorType (LLVMFloatType (), 4);
292 } else if (!strcmp (klass->name, "Vector8s")) {
293 return LLVMVectorType (LLVMInt16Type (), 8);
294 } else if (!strcmp (klass->name, "Vector8us")) {
295 return LLVMVectorType (LLVMInt16Type (), 8);
296 } else if (!strcmp (klass->name, "Vector16sb")) {
297 return LLVMVectorType (LLVMInt8Type (), 16);
298 } else if (!strcmp (klass->name, "Vector16b")) {
299 return LLVMVectorType (LLVMInt8Type (), 16);
301 printf ("%s\n", klass->name);
307 /* Return the 128 bit SIMD type corresponding to the mono type TYPE */
308 static inline G_GNUC_UNUSED LLVMTypeRef
309 type_to_simd_type (int type)
313 return LLVMVectorType (LLVMInt8Type (), 16);
315 return LLVMVectorType (LLVMInt16Type (), 8);
317 return LLVMVectorType (LLVMInt32Type (), 4);
319 return LLVMVectorType (LLVMInt64Type (), 2);
321 return LLVMVectorType (LLVMDoubleType (), 2);
323 return LLVMVectorType (LLVMFloatType (), 4);
325 g_assert_not_reached ();
331 create_llvm_type_for_type (MonoClass *klass)
333 int i, size, nfields, esize;
334 LLVMTypeRef *eltypes;
339 t = &klass->byval_arg;
341 if (mini_type_is_hfa (t, &nfields, &esize)) {
343 * This is needed on arm64 where HFAs are returned in
347 eltypes = g_new (LLVMTypeRef, size);
348 for (i = 0; i < size; ++i)
349 eltypes [i] = esize == 4 ? LLVMFloatType () : LLVMDoubleType ();
351 size = get_vtype_size (t);
353 eltypes = g_new (LLVMTypeRef, size);
354 for (i = 0; i < size; ++i)
355 eltypes [i] = LLVMInt8Type ();
358 name = mono_type_full_name (&klass->byval_arg);
359 ltype = LLVMStructCreateNamed (LLVMGetGlobalContext (), name);
360 LLVMStructSetBody (ltype, eltypes, size, FALSE);
370 * Return the LLVM type corresponding to T.
373 type_to_llvm_type (EmitContext *ctx, MonoType *t)
375 t = mini_replace_type (t);
378 return LLVMPointerType (LLVMInt8Type (), 0);
381 return LLVMVoidType ();
383 return LLVMInt8Type ();
385 return LLVMInt16Type ();
387 return LLVMInt32Type ();
389 return LLVMInt8Type ();
391 return LLVMInt16Type ();
393 return LLVMInt32Type ();
394 case MONO_TYPE_BOOLEAN:
395 return LLVMInt8Type ();
398 return LLVMInt64Type ();
400 return LLVMInt16Type ();
402 return LLVMFloatType ();
404 return LLVMDoubleType ();
407 return IntPtrType ();
408 case MONO_TYPE_OBJECT:
409 case MONO_TYPE_CLASS:
410 case MONO_TYPE_ARRAY:
411 case MONO_TYPE_SZARRAY:
412 case MONO_TYPE_STRING:
414 return ObjRefType ();
417 /* Because of generic sharing */
418 return ObjRefType ();
419 case MONO_TYPE_GENERICINST:
420 if (!mono_type_generic_inst_is_valuetype (t))
421 return ObjRefType ();
423 case MONO_TYPE_VALUETYPE:
424 case MONO_TYPE_TYPEDBYREF: {
428 klass = mono_class_from_mono_type (t);
430 if (MONO_CLASS_IS_SIMD (ctx->cfg, klass))
431 return simd_class_to_llvm_type (ctx, klass);
434 return type_to_llvm_type (ctx, mono_class_enum_basetype (klass));
436 ltype = g_hash_table_lookup (ctx->lmodule->llvm_types, klass);
438 ltype = create_llvm_type_for_type (klass);
439 g_hash_table_insert (ctx->lmodule->llvm_types, klass, ltype);
445 printf ("X: %d\n", t->type);
446 ctx->cfg->exception_message = g_strdup_printf ("type %s", mono_type_full_name (t));
447 ctx->cfg->disable_llvm = TRUE;
455 * Return whenever T is an unsigned int type.
458 type_is_unsigned (EmitContext *ctx, MonoType *t)
475 * type_to_llvm_arg_type:
477 * Same as type_to_llvm_type, but treat i8/i16 as i32.
480 type_to_llvm_arg_type (EmitContext *ctx, MonoType *t)
482 LLVMTypeRef ptype = type_to_llvm_type (ctx, t);
485 * This works on all abis except arm64/ios which passes multiple
486 * arguments in one stack slot.
489 if (ptype == LLVMInt8Type () || ptype == LLVMInt16Type ()) {
491 * LLVM generates code which only sets the lower bits, while JITted
492 * code expects all the bits to be set.
494 ptype = LLVMInt32Type ();
502 * llvm_type_to_stack_type:
504 * Return the LLVM type which needs to be used when a value of type TYPE is pushed
507 static G_GNUC_UNUSED LLVMTypeRef
508 llvm_type_to_stack_type (MonoCompile *cfg, LLVMTypeRef type)
512 if (type == LLVMInt8Type ())
513 return LLVMInt32Type ();
514 else if (type == LLVMInt16Type ())
515 return LLVMInt32Type ();
516 else if (!cfg->r4fp && type == LLVMFloatType ())
517 return LLVMDoubleType ();
523 * regtype_to_llvm_type:
525 * Return the LLVM type corresponding to the regtype C used in instruction
529 regtype_to_llvm_type (char c)
533 return LLVMInt32Type ();
535 return LLVMInt64Type ();
537 return LLVMDoubleType ();
546 * Return the LLVM type corresponding to the unary/binary opcode OPCODE.
549 op_to_llvm_type (int opcode)
554 return LLVMInt8Type ();
557 return LLVMInt8Type ();
560 return LLVMInt16Type ();
563 return LLVMInt16Type ();
566 return LLVMInt32Type ();
569 return LLVMInt32Type ();
571 return LLVMInt64Type ();
573 return LLVMFloatType ();
575 return LLVMDoubleType ();
577 return LLVMInt64Type ();
579 return LLVMInt32Type ();
581 return LLVMInt64Type ();
586 return LLVMInt8Type ();
591 return LLVMInt16Type ();
594 return sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type ();
601 return LLVMInt32Type ();
608 return LLVMInt64Type ();
610 printf ("%s\n", mono_inst_name (opcode));
611 g_assert_not_reached ();
617 * load_store_to_llvm_type:
619 * Return the size/sign/zero extension corresponding to the load/store opcode
623 load_store_to_llvm_type (int opcode, int *size, gboolean *sext, gboolean *zext)
629 case OP_LOADI1_MEMBASE:
630 case OP_STOREI1_MEMBASE_REG:
631 case OP_STOREI1_MEMBASE_IMM:
632 case OP_ATOMIC_LOAD_I1:
633 case OP_ATOMIC_STORE_I1:
636 return LLVMInt8Type ();
637 case OP_LOADU1_MEMBASE:
639 case OP_ATOMIC_LOAD_U1:
640 case OP_ATOMIC_STORE_U1:
643 return LLVMInt8Type ();
644 case OP_LOADI2_MEMBASE:
645 case OP_STOREI2_MEMBASE_REG:
646 case OP_STOREI2_MEMBASE_IMM:
647 case OP_ATOMIC_LOAD_I2:
648 case OP_ATOMIC_STORE_I2:
651 return LLVMInt16Type ();
652 case OP_LOADU2_MEMBASE:
654 case OP_ATOMIC_LOAD_U2:
655 case OP_ATOMIC_STORE_U2:
658 return LLVMInt16Type ();
659 case OP_LOADI4_MEMBASE:
660 case OP_LOADU4_MEMBASE:
663 case OP_STOREI4_MEMBASE_REG:
664 case OP_STOREI4_MEMBASE_IMM:
665 case OP_ATOMIC_LOAD_I4:
666 case OP_ATOMIC_STORE_I4:
667 case OP_ATOMIC_LOAD_U4:
668 case OP_ATOMIC_STORE_U4:
670 return LLVMInt32Type ();
671 case OP_LOADI8_MEMBASE:
673 case OP_STOREI8_MEMBASE_REG:
674 case OP_STOREI8_MEMBASE_IMM:
675 case OP_ATOMIC_LOAD_I8:
676 case OP_ATOMIC_STORE_I8:
677 case OP_ATOMIC_LOAD_U8:
678 case OP_ATOMIC_STORE_U8:
680 return LLVMInt64Type ();
681 case OP_LOADR4_MEMBASE:
682 case OP_STORER4_MEMBASE_REG:
683 case OP_ATOMIC_LOAD_R4:
684 case OP_ATOMIC_STORE_R4:
686 return LLVMFloatType ();
687 case OP_LOADR8_MEMBASE:
688 case OP_STORER8_MEMBASE_REG:
689 case OP_ATOMIC_LOAD_R8:
690 case OP_ATOMIC_STORE_R8:
692 return LLVMDoubleType ();
693 case OP_LOAD_MEMBASE:
695 case OP_STORE_MEMBASE_REG:
696 case OP_STORE_MEMBASE_IMM:
697 *size = sizeof (gpointer);
698 return IntPtrType ();
700 g_assert_not_reached ();
708 * Return the LLVM intrinsics corresponding to the overflow opcode OPCODE.
711 ovf_op_to_intrins (int opcode)
715 return "llvm.sadd.with.overflow.i32";
717 return "llvm.uadd.with.overflow.i32";
719 return "llvm.ssub.with.overflow.i32";
721 return "llvm.usub.with.overflow.i32";
723 return "llvm.smul.with.overflow.i32";
725 return "llvm.umul.with.overflow.i32";
727 return "llvm.sadd.with.overflow.i64";
729 return "llvm.uadd.with.overflow.i64";
731 return "llvm.ssub.with.overflow.i64";
733 return "llvm.usub.with.overflow.i64";
735 return "llvm.smul.with.overflow.i64";
737 return "llvm.umul.with.overflow.i64";
739 g_assert_not_reached ();
745 simd_op_to_intrins (int opcode)
748 #if defined(TARGET_X86) || defined(TARGET_AMD64)
750 return "llvm.x86.sse2.min.pd";
752 return "llvm.x86.sse.min.ps";
754 return "llvm.x86.sse41.pminud";
756 return "llvm.x86.sse41.pminuw";
758 return "llvm.x86.sse2.pminu.b";
760 return "llvm.x86.sse2.pmins.w";
762 return "llvm.x86.sse2.max.pd";
764 return "llvm.x86.sse.max.ps";
766 return "llvm.x86.sse3.hadd.pd";
768 return "llvm.x86.sse3.hadd.ps";
770 return "llvm.x86.sse3.hsub.pd";
772 return "llvm.x86.sse3.hsub.ps";
774 return "llvm.x86.sse41.pmaxud";
776 return "llvm.x86.sse41.pmaxuw";
778 return "llvm.x86.sse2.pmaxu.b";
780 return "llvm.x86.sse3.addsub.ps";
782 return "llvm.x86.sse3.addsub.pd";
783 case OP_EXTRACT_MASK:
784 return "llvm.x86.sse2.pmovmskb.128";
787 return "llvm.x86.sse2.psrli.w";
790 return "llvm.x86.sse2.psrli.d";
793 return "llvm.x86.sse2.psrli.q";
796 return "llvm.x86.sse2.pslli.w";
799 return "llvm.x86.sse2.pslli.d";
802 return "llvm.x86.sse2.pslli.q";
805 return "llvm.x86.sse2.psrai.w";
808 return "llvm.x86.sse2.psrai.d";
810 return "llvm.x86.sse2.padds.b";
812 return "llvm.x86.sse2.padds.w";
814 return "llvm.x86.sse2.psubs.b";
816 return "llvm.x86.sse2.psubs.w";
817 case OP_PADDB_SAT_UN:
818 return "llvm.x86.sse2.paddus.b";
819 case OP_PADDW_SAT_UN:
820 return "llvm.x86.sse2.paddus.w";
821 case OP_PSUBB_SAT_UN:
822 return "llvm.x86.sse2.psubus.b";
823 case OP_PSUBW_SAT_UN:
824 return "llvm.x86.sse2.psubus.w";
826 return "llvm.x86.sse2.pavg.b";
828 return "llvm.x86.sse2.pavg.w";
830 return "llvm.x86.sse.sqrt.ps";
832 return "llvm.x86.sse2.sqrt.pd";
834 return "llvm.x86.sse.rsqrt.ps";
836 return "llvm.x86.sse.rcp.ps";
838 return "llvm.x86.sse2.cvtdq2pd";
840 return "llvm.x86.sse2.cvtdq2ps";
842 return "llvm.x86.sse2.cvtpd2dq";
844 return "llvm.x86.sse2.cvtps2dq";
846 return "llvm.x86.sse2.cvtpd2ps";
848 return "llvm.x86.sse2.cvtps2pd";
850 return "llvm.x86.sse2.cvttpd2dq";
852 return "llvm.x86.sse2.cvttps2dq";
854 return "llvm.x86.sse.cmp.ps";
856 return "llvm.x86.sse2.cmp.pd";
858 return "llvm.x86.sse2.packsswb.128";
860 return "llvm.x86.sse2.packssdw.128";
862 return "llvm.x86.sse2.packuswb.128";
864 return "llvm.x86.sse41.packusdw";
866 return "llvm.x86.sse2.pmulh.w";
867 case OP_PMULW_HIGH_UN:
868 return "llvm.x86.sse2.pmulhu.w";
871 g_assert_not_reached ();
877 simd_op_to_llvm_type (int opcode)
879 #if defined(TARGET_X86) || defined(TARGET_AMD64)
883 return type_to_simd_type (MONO_TYPE_R8);
886 return type_to_simd_type (MONO_TYPE_I8);
889 return type_to_simd_type (MONO_TYPE_I4);
894 return type_to_simd_type (MONO_TYPE_I2);
898 return type_to_simd_type (MONO_TYPE_I1);
900 return type_to_simd_type (MONO_TYPE_R4);
903 return type_to_simd_type (MONO_TYPE_I4);
907 return type_to_simd_type (MONO_TYPE_R8);
911 return type_to_simd_type (MONO_TYPE_R4);
912 case OP_EXTRACT_MASK:
913 return type_to_simd_type (MONO_TYPE_I1);
919 return type_to_simd_type (MONO_TYPE_R4);
922 return type_to_simd_type (MONO_TYPE_R8);
924 g_assert_not_reached ();
935 * Return the LLVM basic block corresponding to BB.
937 static LLVMBasicBlockRef
938 get_bb (EmitContext *ctx, MonoBasicBlock *bb)
940 char bb_name_buf [128];
943 if (ctx->bblocks [bb->block_num].bblock == NULL) {
944 if (bb->flags & BB_EXCEPTION_HANDLER) {
945 int clause_index = (mono_get_block_region_notry (ctx->cfg, bb->region) >> 8) - 1;
946 sprintf (bb_name_buf, "EH_CLAUSE%d_BB%d", clause_index, bb->block_num);
947 bb_name = bb_name_buf;
948 } else if (bb->block_num < 256) {
949 if (!ctx->lmodule->bb_names) {
950 ctx->lmodule->bb_names_len = 256;
951 ctx->lmodule->bb_names = g_new0 (char*, ctx->lmodule->bb_names_len);
953 if (!ctx->lmodule->bb_names [bb->block_num]) {
956 n = g_strdup_printf ("BB%d", bb->block_num);
957 mono_memory_barrier ();
958 ctx->lmodule->bb_names [bb->block_num] = n;
960 bb_name = ctx->lmodule->bb_names [bb->block_num];
962 sprintf (bb_name_buf, "BB%d", bb->block_num);
963 bb_name = bb_name_buf;
966 ctx->bblocks [bb->block_num].bblock = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
967 ctx->bblocks [bb->block_num].end_bblock = ctx->bblocks [bb->block_num].bblock;
970 return ctx->bblocks [bb->block_num].bblock;
976 * Return the last LLVM bblock corresponding to BB.
977 * This might not be equal to the bb returned by get_bb () since we need to generate
978 * multiple LLVM bblocks for a mono bblock to handle throwing exceptions.
980 static LLVMBasicBlockRef
981 get_end_bb (EmitContext *ctx, MonoBasicBlock *bb)
984 return ctx->bblocks [bb->block_num].end_bblock;
987 static LLVMBasicBlockRef
988 gen_bb (EmitContext *ctx, const char *prefix)
992 sprintf (bb_name, "%s%d", prefix, ++ ctx->ex_index);
993 return LLVMAppendBasicBlock (ctx->lmethod, bb_name);
999 * Return the target of the patch identified by TYPE and TARGET.
1002 resolve_patch (MonoCompile *cfg, MonoJumpInfoType type, gconstpointer target)
1006 memset (&ji, 0, sizeof (ji));
1008 ji.data.target = target;
1010 return mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
1016 * Emit code to convert the LLVM value V to DTYPE.
1019 convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_unsigned)
1021 LLVMTypeRef stype = LLVMTypeOf (v);
1023 if (stype != dtype) {
1024 gboolean ext = FALSE;
1027 if (dtype == LLVMInt64Type () && (stype == LLVMInt32Type () || stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1029 else if (dtype == LLVMInt32Type () && (stype == LLVMInt16Type () || stype == LLVMInt8Type ()))
1031 else if (dtype == LLVMInt16Type () && (stype == LLVMInt8Type ()))
1035 return is_unsigned ? LLVMBuildZExt (ctx->builder, v, dtype, "") : LLVMBuildSExt (ctx->builder, v, dtype, "");
1037 if (dtype == LLVMDoubleType () && stype == LLVMFloatType ())
1038 return LLVMBuildFPExt (ctx->builder, v, dtype, "");
1041 if (stype == LLVMInt64Type () && (dtype == LLVMInt32Type () || dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1042 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1043 if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ()))
1044 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1045 if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ())
1046 return LLVMBuildTrunc (ctx->builder, v, dtype, "");
1047 if (stype == LLVMDoubleType () && dtype == LLVMFloatType ())
1048 return LLVMBuildFPTrunc (ctx->builder, v, dtype, "");
1050 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind && LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1051 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1052 if (LLVMGetTypeKind (dtype) == LLVMPointerTypeKind)
1053 return LLVMBuildIntToPtr (ctx->builder, v, dtype, "");
1054 if (LLVMGetTypeKind (stype) == LLVMPointerTypeKind)
1055 return LLVMBuildPtrToInt (ctx->builder, v, dtype, "");
1057 if (mono_arch_is_soft_float ()) {
1058 if (stype == LLVMInt32Type () && dtype == LLVMFloatType ())
1059 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1060 if (stype == LLVMInt32Type () && dtype == LLVMDoubleType ())
1061 return LLVMBuildBitCast (ctx->builder, LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""), dtype, "");
1064 if (LLVMGetTypeKind (stype) == LLVMVectorTypeKind && LLVMGetTypeKind (dtype) == LLVMVectorTypeKind)
1065 return LLVMBuildBitCast (ctx->builder, v, dtype, "");
1068 LLVMDumpValue (LLVMConstNull (dtype));
1069 g_assert_not_reached ();
1077 convert (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype)
1079 return convert_full (ctx, v, dtype, FALSE);
1083 * emit_volatile_load:
1085 * If vreg is volatile, emit a load from its address.
1088 emit_volatile_load (EmitContext *ctx, int vreg)
1092 LLVMValueRef v = LLVMBuildLoad (ctx->builder, ctx->addresses [vreg], "");
1093 t = ctx->vreg_cli_types [vreg];
1094 if (t && !t->byref) {
1096 * Might have to zero extend since llvm doesn't have
1099 if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN)
1100 v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), "");
1101 else if (t->type == MONO_TYPE_U8)
1102 v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), "");
1109 * emit_volatile_store:
1111 * If VREG is volatile, emit a store from its value to its address.
1114 emit_volatile_store (EmitContext *ctx, int vreg)
1116 MonoInst *var = get_vreg_to_inst (ctx->cfg, vreg);
1118 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
1119 g_assert (ctx->addresses [vreg]);
1120 LLVMBuildStore (ctx->builder, convert (ctx, ctx->values [vreg], type_to_llvm_type (ctx, var->inst_vtype)), ctx->addresses [vreg]);
1126 * Maps parameter indexes in the original signature to parameter indexes
1127 * in the LLVM signature.
1130 /* The indexes of various special arguments in the LLVM signature */
1131 int vret_arg_pindex, this_arg_pindex, rgctx_arg_pindex, imt_arg_pindex;
1135 * sig_to_llvm_sig_full:
1137 * Return the LLVM signature corresponding to the mono signature SIG using the
1138 * calling convention information in CINFO. Return parameter mapping information in SINFO.
1141 sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo,
1144 LLVMTypeRef ret_type;
1145 LLVMTypeRef *param_types = NULL;
1147 int i, j, pindex, vret_arg_pindex = 0;
1149 gboolean vretaddr = FALSE;
1153 memset (sinfo, 0, sizeof (LLVMSigInfo));
1155 rtype = mini_replace_type (sig->ret);
1156 ret_type = type_to_llvm_type (ctx, rtype);
1157 CHECK_FAILURE (ctx);
1160 if (cinfo->ret.storage == LLVMArgVtypeInReg) {
1161 /* LLVM models this by returning an aggregate value */
1162 if (cinfo->ret.pair_storage [0] == LLVMArgInIReg && cinfo->ret.pair_storage [1] == LLVMArgNone) {
1163 LLVMTypeRef members [2];
1165 members [0] = IntPtrType ();
1166 ret_type = LLVMStructType (members, 1, FALSE);
1168 g_assert_not_reached ();
1170 } else if (cinfo->ret.storage == LLVMArgVtypeByVal) {
1171 /* Vtype returned normally by val */
1172 } else if (cinfo->ret.storage == LLVMArgFpStruct) {
1173 /* Vtype returned as a fp struct */
1174 LLVMTypeRef members [16];
1176 /* Have to create our own structure since we don't map fp structures to LLVM fp structures yet */
1177 for (i = 0; i < cinfo->ret.nslots; ++i)
1178 members [i] = cinfo->ret.esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1179 ret_type = LLVMStructType (members, cinfo->ret.nslots, FALSE);
1180 } else if (mini_type_is_vtype (ctx->cfg, rtype)) {
1181 g_assert (cinfo->ret.storage == LLVMArgVtypeRetAddr);
1183 ret_type = LLVMVoidType ();
1187 pindexes = g_new0 (int, sig->param_count);
1188 param_types = g_new0 (LLVMTypeRef, (sig->param_count * 8) + 3);
1190 if (cinfo && cinfo->rgctx_arg) {
1192 sinfo->rgctx_arg_pindex = pindex;
1193 param_types [pindex] = ctx->lmodule->ptr_type;
1196 if (cinfo && cinfo->imt_arg) {
1198 sinfo->imt_arg_pindex = pindex;
1199 param_types [pindex] = ctx->lmodule->ptr_type;
1203 /* Compute the index in the LLVM signature where the vret arg needs to be passed */
1204 vret_arg_pindex = pindex;
1205 if (cinfo->vret_arg_index == 1) {
1206 /* Add the slots consumed by the first argument */
1207 LLVMArgInfo *ainfo = &cinfo->args [0];
1208 switch (ainfo->storage) {
1209 case LLVMArgVtypeInReg:
1210 for (j = 0; j < 2; ++j) {
1211 if (ainfo->pair_storage [j] == LLVMArgInIReg)
1221 sinfo->vret_arg_pindex = vret_arg_pindex;
1224 if (vretaddr && vret_arg_pindex == pindex)
1225 param_types [pindex ++] = IntPtrType ();
1228 sinfo->this_arg_pindex = pindex;
1229 param_types [pindex ++] = ThisType ();
1231 if (vretaddr && vret_arg_pindex == pindex)
1232 param_types [pindex ++] = IntPtrType ();
1233 for (i = 0; i < sig->param_count; ++i) {
1234 LLVMArgInfo *ainfo = cinfo ? &cinfo->args [i + sig->hasthis] : NULL;
1236 if (vretaddr && vret_arg_pindex == pindex)
1237 param_types [pindex ++] = IntPtrType ();
1238 pindexes [i] = pindex;
1241 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1245 switch (ainfo->storage) {
1246 case LLVMArgVtypeInReg:
1247 for (j = 0; j < 2; ++j) {
1248 switch (ainfo->pair_storage [j]) {
1250 param_types [pindex ++] = LLVMIntType (sizeof (gpointer) * 8);
1255 g_assert_not_reached ();
1259 case LLVMArgVtypeByVal:
1260 param_types [pindex] = type_to_llvm_arg_type (ctx, sig->params [i]);
1261 CHECK_FAILURE (ctx);
1262 param_types [pindex] = LLVMPointerType (param_types [pindex], 0);
1265 case LLVMArgAsIArgs:
1266 param_types [pindex] = LLVMArrayType (IntPtrType (), ainfo->nslots);
1269 case LLVMArgAsFpArgs: {
1272 for (j = 0; j < ainfo->nslots; ++j)
1273 param_types [pindex + j] = ainfo->esize == 8 ? LLVMDoubleType () : LLVMFloatType ();
1274 pindex += ainfo->nslots;
1278 param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]);
1282 if (vretaddr && vret_arg_pindex == pindex)
1283 param_types [pindex ++] = IntPtrType ();
1285 CHECK_FAILURE (ctx);
1287 res = LLVMFunctionType (ret_type, param_types, pindex, FALSE);
1288 g_free (param_types);
1291 sinfo->pindexes = pindexes;
1299 g_free (param_types);
1305 sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig)
1307 return sig_to_llvm_sig_full (ctx, sig, NULL, NULL);
1311 * LLVMFunctionType1:
1313 * Create an LLVM function type from the arguments.
1315 static G_GNUC_UNUSED LLVMTypeRef
1316 LLVMFunctionType1(LLVMTypeRef ReturnType,
1317 LLVMTypeRef ParamType1,
1320 LLVMTypeRef param_types [1];
1322 param_types [0] = ParamType1;
1324 return LLVMFunctionType (ReturnType, param_types, 1, IsVarArg);
1328 * LLVMFunctionType2:
1330 * Create an LLVM function type from the arguments.
1332 static G_GNUC_UNUSED LLVMTypeRef
1333 LLVMFunctionType2(LLVMTypeRef ReturnType,
1334 LLVMTypeRef ParamType1,
1335 LLVMTypeRef ParamType2,
1338 LLVMTypeRef param_types [2];
1340 param_types [0] = ParamType1;
1341 param_types [1] = ParamType2;
1343 return LLVMFunctionType (ReturnType, param_types, 2, IsVarArg);
1347 * LLVMFunctionType3:
1349 * Create an LLVM function type from the arguments.
1351 static G_GNUC_UNUSED LLVMTypeRef
1352 LLVMFunctionType3(LLVMTypeRef ReturnType,
1353 LLVMTypeRef ParamType1,
1354 LLVMTypeRef ParamType2,
1355 LLVMTypeRef ParamType3,
1358 LLVMTypeRef param_types [3];
1360 param_types [0] = ParamType1;
1361 param_types [1] = ParamType2;
1362 param_types [2] = ParamType3;
1364 return LLVMFunctionType (ReturnType, param_types, 3, IsVarArg);
1370 * Create an LLVM builder and remember it so it can be freed later.
1372 static LLVMBuilderRef
1373 create_builder (EmitContext *ctx)
1375 LLVMBuilderRef builder = LLVMCreateBuilder ();
1377 ctx->builders = g_slist_prepend_mempool (ctx->cfg->mempool, ctx->builders, builder);
1383 get_plt_entry (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gconstpointer data)
1385 char *callee_name = mono_aot_get_plt_symbol (type, data);
1386 LLVMValueRef callee;
1387 MonoJumpInfo *ji = NULL;
1392 if (ctx->cfg->compile_aot)
1393 /* Add a patch so referenced wrappers can be compiled in full aot mode */
1394 mono_add_patch_info (ctx->cfg, 0, type, data);
1397 callee = g_hash_table_lookup (ctx->lmodule->plt_entries, callee_name);
1399 callee = LLVMAddFunction (ctx->module, callee_name, llvm_sig);
1401 LLVMSetVisibility (callee, LLVMHiddenVisibility);
1403 g_hash_table_insert (ctx->lmodule->plt_entries, (char*)callee_name, callee);
1406 if (ctx->cfg->compile_aot) {
1407 ji = g_new0 (MonoJumpInfo, 1);
1409 ji->data.target = data;
1411 g_hash_table_insert (ctx->lmodule->plt_entries_ji, ji, callee);
1418 get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb)
1420 MonoMethodHeader *header = cfg->header;
1421 MonoExceptionClause *clause;
1425 if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))
1426 return (bb->region >> 8) - 1;
1429 for (i = 0; i < header->num_clauses; ++i) {
1430 clause = &header->clauses [i];
1432 if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE)
1440 set_metadata_flag (LLVMValueRef v, const char *flag_name)
1442 LLVMValueRef md_arg;
1445 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1446 md_arg = LLVMMDString ("mono", 4);
1447 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1451 set_invariant_load_flag (LLVMValueRef v)
1453 LLVMValueRef md_arg;
1455 const char *flag_name;
1457 // FIXME: Cache this
1458 flag_name = "invariant.load";
1459 md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name));
1460 md_arg = LLVMMDString ("<index>", strlen ("<index>"));
1461 LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1));
1467 * Emit an LLVM call or invoke instruction depending on whenever the call is inside
1471 emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LLVMValueRef callee, LLVMValueRef *args, int pindex)
1473 MonoCompile *cfg = ctx->cfg;
1475 LLVMBuilderRef builder = *builder_ref;
1478 clause_index = get_handler_clause (cfg, bb);
1480 if (clause_index != -1) {
1481 MonoMethodHeader *header = cfg->header;
1482 MonoExceptionClause *ec = &header->clauses [clause_index];
1483 MonoBasicBlock *tblock;
1484 LLVMBasicBlockRef ex_bb, noex_bb;
1487 * Have to use an invoke instead of a call, branching to the
1488 * handler bblock of the clause containing this bblock.
1491 g_assert (ec->flags == MONO_EXCEPTION_CLAUSE_NONE || ec->flags == MONO_EXCEPTION_CLAUSE_FINALLY);
1493 tblock = cfg->cil_offset_to_bb [ec->handler_offset];
1496 ctx->bblocks [tblock->block_num].invoke_target = TRUE;
1498 ex_bb = get_bb (ctx, tblock);
1500 noex_bb = gen_bb (ctx, "NOEX_BB");
1503 lcall = LLVMBuildInvoke (builder, callee, args, pindex, noex_bb, ex_bb, "");
1505 builder = ctx->builder = create_builder (ctx);
1506 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1508 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1510 lcall = LLVMBuildCall (builder, callee, args, pindex, "");
1511 ctx->builder = builder;
1514 *builder_ref = ctx->builder;
1519 #if LLVM_API_VERSION >= 4
1520 #define EXTRA_MONO_LOAD_STORE_ARGS 1
1522 #define EXTRA_MONO_LOAD_STORE_ARGS 0
1526 emit_load_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting, BarrierKind barrier)
1528 const char *intrins_name;
1529 LLVMValueRef args [16], res;
1530 LLVMTypeRef addr_type;
1532 if (is_faulting && bb->region != -1) {
1533 #if LLVM_API_VERSION >= 4
1534 LLVMAtomicOrdering ordering;
1537 case LLVM_BARRIER_NONE:
1538 ordering = LLVMAtomicOrderingNotAtomic;
1540 case LLVM_BARRIER_ACQ:
1541 ordering = LLVMAtomicOrderingAcquire;
1543 case LLVM_BARRIER_SEQ:
1544 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1547 g_assert_not_reached ();
1553 * We handle loads which can fault by calling a mono specific intrinsic
1554 * using an invoke, so they are handled properly inside try blocks.
1555 * We can't use this outside clauses, since LLVM optimizes intrinsics which
1556 * are marked with IntrReadArgMem.
1560 intrins_name = "llvm.mono.load.i8.p0i8";
1563 intrins_name = "llvm.mono.load.i16.p0i16";
1566 intrins_name = "llvm.mono.load.i32.p0i32";
1569 intrins_name = "llvm.mono.load.i64.p0i64";
1572 g_assert_not_reached ();
1575 addr_type = LLVMTypeOf (addr);
1576 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0))
1577 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1580 args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1581 args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1582 #if LLVM_API_VERSION >= 4
1583 args [3] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1585 res = emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->module, intrins_name), args, 3 + EXTRA_MONO_LOAD_STORE_ARGS);
1587 if (addr_type == LLVMPointerType (LLVMDoubleType (), 0))
1588 res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), "");
1589 else if (addr_type == LLVMPointerType (LLVMFloatType (), 0))
1590 res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), "");
1597 * We emit volatile loads for loads which can fault, because otherwise
1598 * LLVM will generate invalid code when encountering a load from a
1601 res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting, barrier);
1603 /* Mark it with a custom metadata */
1606 set_metadata_flag (res, "mono.faulting.load");
1614 emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting)
1616 return emit_load_general (ctx, bb, builder_ref, size, addr, name, is_faulting, LLVM_BARRIER_NONE);
1620 emit_store_general (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting, BarrierKind barrier)
1622 const char *intrins_name;
1623 LLVMValueRef args [16];
1625 if (is_faulting && bb->region != -1) {
1626 #if LLVM_API_VERSION >= 4
1627 LLVMAtomicOrdering ordering;
1630 case LLVM_BARRIER_NONE:
1631 ordering = LLVMAtomicOrderingNotAtomic;
1633 case LLVM_BARRIER_REL:
1634 ordering = LLVMAtomicOrderingRelease;
1636 case LLVM_BARRIER_SEQ:
1637 ordering = LLVMAtomicOrderingSequentiallyConsistent;
1640 g_assert_not_reached ();
1647 intrins_name = "llvm.mono.store.i8.p0i8";
1650 intrins_name = "llvm.mono.store.i16.p0i16";
1653 intrins_name = "llvm.mono.store.i32.p0i32";
1656 intrins_name = "llvm.mono.store.i64.p0i64";
1659 g_assert_not_reached ();
1662 if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) {
1663 value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), "");
1664 addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), "");
1669 args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
1670 args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE);
1671 #if LLVM_API_VERSION >= 4
1672 args [4] = LLVMConstInt (LLVMInt32Type (), ordering, FALSE);
1674 emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->module, intrins_name), args, 4 + EXTRA_MONO_LOAD_STORE_ARGS);
1676 mono_llvm_build_store (*builder_ref, value, addr, is_faulting, barrier);
1681 emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting)
1683 emit_store_general (ctx, bb, builder_ref, size, value, addr, is_faulting, LLVM_BARRIER_NONE);
1687 * emit_cond_system_exception:
1689 * Emit code to throw the exception EXC_TYPE if the condition CMP is false.
1690 * Might set the ctx exception.
1693 emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp)
1695 LLVMBasicBlockRef ex_bb, noex_bb;
1696 LLVMBuilderRef builder;
1697 MonoClass *exc_class;
1698 LLVMValueRef args [2];
1700 ex_bb = gen_bb (ctx, "EX_BB");
1701 noex_bb = gen_bb (ctx, "NOEX_BB");
1703 LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb);
1705 exc_class = mono_class_from_name (mono_get_corlib (), "System", exc_type);
1706 g_assert (exc_class);
1708 /* Emit exception throwing code */
1709 builder = create_builder (ctx);
1710 LLVMPositionBuilderAtEnd (builder, ex_bb);
1712 if (!ctx->lmodule->throw_corlib_exception) {
1713 LLVMValueRef callee;
1715 const char *icall_name;
1717 MonoMethodSignature *throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 2);
1718 throw_sig->ret = &mono_get_void_class ()->byval_arg;
1719 throw_sig->params [0] = &mono_get_int32_class ()->byval_arg;
1720 icall_name = "llvm_throw_corlib_exception_abs_trampoline";
1721 /* This will become i8* */
1722 throw_sig->params [1] = &mono_get_byte_class ()->this_arg;
1723 sig = sig_to_llvm_sig (ctx, throw_sig);
1725 if (ctx->cfg->compile_aot) {
1726 callee = get_plt_entry (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
1728 callee = LLVMAddFunction (ctx->module, "llvm_throw_corlib_exception_trampoline", sig_to_llvm_sig (ctx, throw_sig));
1731 * Differences between the LLVM/non-LLVM throw corlib exception trampoline:
1732 * - On x86, LLVM generated code doesn't push the arguments
1733 * - The trampoline takes the throw address as an arguments, not a pc offset.
1735 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
1738 mono_memory_barrier ();
1739 ctx->lmodule->throw_corlib_exception = callee;
1742 if (IS_TARGET_X86 || IS_TARGET_AMD64)
1743 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE);
1745 args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE);
1748 * The LLVM mono branch contains changes so a block address can be passed as an
1749 * argument to a call.
1751 args [1] = LLVMBlockAddress (ctx->lmethod, ex_bb);
1752 emit_call (ctx, bb, &builder, ctx->lmodule->throw_corlib_exception, args, 2);
1754 LLVMBuildUnreachable (builder);
1756 ctx->builder = create_builder (ctx);
1757 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
1759 ctx->bblocks [bb->block_num].end_bblock = noex_bb;
1766 * emit_args_to_vtype:
1768 * Emit code to store the vtype in the arguments args to the address ADDRESS.
1771 emit_args_to_vtype (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args)
1773 int j, size, nslots;
1775 size = get_vtype_size (t);
1777 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1778 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
1781 if (ainfo->storage == LLVMArgAsFpArgs)
1782 nslots = ainfo->nslots;
1786 for (j = 0; j < nslots; ++j) {
1787 LLVMValueRef index [2], addr, daddr;
1788 int part_size = size > sizeof (gpointer) ? sizeof (gpointer) : size;
1789 LLVMTypeRef part_type;
1791 if (ainfo->pair_storage [j] == LLVMArgNone)
1794 switch (ainfo->pair_storage [j]) {
1795 case LLVMArgInIReg: {
1796 part_type = LLVMIntType (part_size * 8);
1797 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1798 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1799 addr = LLVMBuildGEP (builder, address, index, 1, "");
1801 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
1802 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1803 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1805 LLVMBuildStore (builder, convert (ctx, args [j], part_type), LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (part_type, 0), ""));
1808 case LLVMArgInFPReg: {
1809 LLVMTypeRef arg_type;
1811 if (ainfo->esize == 8)
1812 arg_type = LLVMDoubleType ();
1814 arg_type = LLVMFloatType ();
1816 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1817 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
1818 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1819 LLVMBuildStore (builder, args [j], addr);
1825 g_assert_not_reached ();
1828 size -= sizeof (gpointer);
1833 * emit_vtype_to_args:
1835 * Emit code to load a vtype at address ADDRESS into scalar arguments. Store the arguments
1836 * into ARGS, and the number of arguments into NARGS.
1839 emit_vtype_to_args (EmitContext *ctx, LLVMBuilderRef builder, MonoType *t, LLVMValueRef address, LLVMArgInfo *ainfo, LLVMValueRef *args, guint32 *nargs)
1842 int j, size, nslots;
1843 LLVMTypeRef arg_type;
1845 size = get_vtype_size (t);
1847 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t)))
1848 address = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (LLVMInt8Type (), 0), "");
1850 if (ainfo->storage == LLVMArgAsFpArgs)
1851 nslots = ainfo->nslots;
1854 for (j = 0; j < nslots; ++j) {
1855 LLVMValueRef index [2], addr, daddr;
1856 int partsize = size > sizeof (gpointer) ? sizeof (gpointer) : size;
1858 if (ainfo->pair_storage [j] == LLVMArgNone)
1861 switch (ainfo->pair_storage [j]) {
1863 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (t))) {
1864 index [0] = LLVMConstInt (LLVMInt32Type (), j * sizeof (gpointer), FALSE);
1865 addr = LLVMBuildGEP (builder, address, index, 1, "");
1867 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (IntPtrType (), 0), "");
1868 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1869 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1871 args [pindex ++] = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (ctx->builder, addr, LLVMPointerType (LLVMIntType (partsize * 8), 0), ""), ""), IntPtrType ());
1873 case LLVMArgInFPReg:
1874 if (ainfo->esize == 8)
1875 arg_type = LLVMDoubleType ();
1877 arg_type = LLVMFloatType ();
1878 daddr = LLVMBuildBitCast (ctx->builder, address, LLVMPointerType (arg_type, 0), "");
1879 index [0] = LLVMConstInt (LLVMInt32Type (), j, FALSE);
1880 addr = LLVMBuildGEP (builder, daddr, index, 1, "");
1881 args [pindex ++] = LLVMBuildLoad (builder, addr, "");
1886 g_assert_not_reached ();
1888 size -= sizeof (gpointer);
1895 build_alloca_llvm_type (EmitContext *ctx, LLVMTypeRef t, int align)
1898 * Have to place all alloca's at the end of the entry bb, since otherwise they would
1899 * get executed every time control reaches them.
1901 LLVMPositionBuilder (ctx->alloca_builder, get_bb (ctx, ctx->cfg->bb_entry), ctx->last_alloca);
1903 ctx->last_alloca = mono_llvm_build_alloca (ctx->alloca_builder, t, NULL, align, "");
1904 return ctx->last_alloca;
1908 build_alloca (EmitContext *ctx, MonoType *t)
1910 MonoClass *k = mono_class_from_mono_type (t);
1913 if (MONO_CLASS_IS_SIMD (ctx->cfg, k))
1916 align = mono_class_min_align (k);
1918 /* Sometimes align is not a power of 2 */
1919 while (mono_is_power_of_two (align) == -1)
1922 return build_alloca_llvm_type (ctx, type_to_llvm_type (ctx, t), align);
1926 * Put the global into the 'llvm.used' array to prevent it from being optimized away.
1929 mark_as_used (MonoLLVMModule *lmodule, LLVMValueRef global)
1932 lmodule->used = g_ptr_array_sized_new (16);
1933 g_ptr_array_add (lmodule->used, global);
1937 emit_llvm_used (MonoLLVMModule *lmodule)
1939 LLVMModuleRef module = lmodule->module;
1940 LLVMTypeRef used_type;
1941 LLVMValueRef used, *used_elem;
1947 used_type = LLVMArrayType (LLVMPointerType (LLVMInt8Type (), 0), lmodule->used->len);
1948 used = LLVMAddGlobal (module, used_type, "llvm.used");
1949 used_elem = g_new0 (LLVMValueRef, lmodule->used->len);
1950 for (i = 0; i < lmodule->used->len; ++i)
1951 used_elem [i] = LLVMConstBitCast (g_ptr_array_index (lmodule->used, i), LLVMPointerType (LLVMInt8Type (), 0));
1952 LLVMSetInitializer (used, LLVMConstArray (LLVMPointerType (LLVMInt8Type (), 0), used_elem, lmodule->used->len));
1953 LLVMSetLinkage (used, LLVMAppendingLinkage);
1954 LLVMSetSection (used, "llvm.metadata");
1960 * Emit code to load/convert arguments.
1963 emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder)
1966 MonoCompile *cfg = ctx->cfg;
1967 MonoMethodSignature *sig = ctx->sig;
1968 LLVMCallInfo *linfo = ctx->linfo;
1971 ctx->alloca_builder = create_builder (ctx);
1974 * Handle indirect/volatile variables by allocating memory for them
1975 * using 'alloca', and storing their address in a temporary.
1977 for (i = 0; i < cfg->num_varinfo; ++i) {
1978 MonoInst *var = cfg->varinfo [i];
1981 if (var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT) || mini_type_is_vtype (cfg, var->inst_vtype)) {
1982 vtype = type_to_llvm_type (ctx, var->inst_vtype);
1983 CHECK_FAILURE (ctx);
1984 /* Could be already created by an OP_VPHI */
1985 if (!ctx->addresses [var->dreg])
1986 ctx->addresses [var->dreg] = build_alloca (ctx, var->inst_vtype);
1987 ctx->vreg_cli_types [var->dreg] = var->inst_vtype;
1991 for (i = 0; i < sig->param_count; ++i) {
1992 LLVMArgInfo *ainfo = &linfo->args [i + sig->hasthis];
1993 int reg = cfg->args [i + sig->hasthis]->dreg;
1995 switch (ainfo->storage) {
1996 case LLVMArgVtypeInReg:
1997 case LLVMArgAsFpArgs: {
1998 LLVMValueRef args [8];
2001 /* The argument is received as a set of int/fp arguments, store them into the real argument */
2002 memset (args, 0, sizeof (args));
2003 pindex = ctx->pindexes [i];
2004 if (ainfo->storage == LLVMArgVtypeInReg) {
2005 args [0] = LLVMGetParam (ctx->lmethod, pindex);
2006 if (ainfo->pair_storage [1] != LLVMArgNone)
2007 args [1] = LLVMGetParam (ctx->lmethod, pindex + 1);
2009 g_assert (ainfo->nslots <= 8);
2010 for (j = 0; j < ainfo->nslots; ++j)
2011 args [j] = LLVMGetParam (ctx->lmethod, ctx->pindexes [i] + j);
2013 ctx->addresses [reg] = build_alloca (ctx, sig->params [i]);
2015 emit_args_to_vtype (ctx, builder, sig->params [i], ctx->addresses [reg], ainfo, args);
2017 if (ainfo->storage == LLVMArgVtypeInReg && MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) {
2018 /* Treat these as normal values */
2019 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2023 case LLVMArgVtypeByVal: {
2024 ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, ctx->pindexes [i]);
2026 if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) {
2027 /* Treat these as normal values */
2028 ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], "");
2032 case LLVMArgAsIArgs: {
2033 LLVMValueRef arg = LLVMGetParam (ctx->lmethod, ctx->pindexes [i]);
2035 ctx->addresses [reg] = build_alloca (ctx, sig->params [i]);
2037 /* The argument is received as an array of ints, store it into the real argument */
2038 LLVMBuildStore (ctx->builder, arg, convert (ctx, ctx->addresses [reg], LLVMPointerType (LLVMTypeOf (arg), 0)));
2042 ctx->values [reg] = convert_full (ctx, ctx->values [reg], llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->params [i])), type_is_unsigned (ctx, sig->params [i]));
2048 emit_volatile_store (ctx, cfg->vret_addr->dreg);
2050 emit_volatile_store (ctx, cfg->args [0]->dreg);
2051 for (i = 0; i < sig->param_count; ++i)
2052 if (!mini_type_is_vtype (cfg, sig->params [i]))
2053 emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg);
2055 if (sig->hasthis && !cfg->rgctx_var && cfg->generic_sharing_context) {
2056 LLVMValueRef this_alloc;
2059 * The exception handling code needs the location where the this argument was
2060 * stored for gshared methods. We create a separate alloca to hold it, and mark it
2061 * with the "mono.this" custom metadata to tell llvm that it needs to save its
2062 * location into the LSDA.
2064 this_alloc = mono_llvm_build_alloca (builder, ThisType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, "");
2065 /* This volatile store will keep the alloca alive */
2066 mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE, LLVM_BARRIER_NONE);
2068 set_metadata_flag (this_alloc, "mono.this");
2071 if (cfg->rgctx_var) {
2072 LLVMValueRef rgctx_alloc, store;
2075 * We handle the rgctx arg similarly to the this pointer.
2077 g_assert (ctx->addresses [cfg->rgctx_var->dreg]);
2078 rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg];
2079 /* This volatile store will keep the alloca alive */
2080 store = mono_llvm_build_store (builder, convert (ctx, ctx->rgctx_arg, IntPtrType ()), rgctx_alloc, TRUE, LLVM_BARRIER_NONE);
2082 set_metadata_flag (rgctx_alloc, "mono.this");
2086 * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever
2087 * it needs to continue normally, or return back to the exception handling system.
2089 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
2090 if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER))
2091 g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb);
2092 if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER) && bb->in_scount == 0) {
2096 sprintf (name, "finally_ind_bb%d", bb->block_num);
2097 val = LLVMBuildAlloca (builder, LLVMInt32Type (), name);
2098 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val);
2100 ctx->bblocks [bb->block_num].finally_ind = val;
2103 * Create a new bblock which CALL_HANDLER can branch to, because branching to the
2104 * LLVM bblock containing the call to llvm.eh.selector causes problems for the
2105 * LLVM optimizer passes.
2107 sprintf (name, "BB_%d_CALL_HANDLER_TARGET", bb->block_num);
2108 ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name);
2116 /* Have to export this for AOT */
2118 mono_personality (void)
2121 g_assert_not_reached ();
2125 process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins)
2127 MonoCompile *cfg = ctx->cfg;
2128 LLVMModuleRef module = ctx->module;
2129 LLVMValueRef *values = ctx->values;
2130 LLVMValueRef *addresses = ctx->addresses;
2131 MonoCallInst *call = (MonoCallInst*)ins;
2132 MonoMethodSignature *sig = call->signature;
2133 LLVMValueRef callee = NULL, lcall;
2135 LLVMCallInfo *cinfo;
2139 LLVMTypeRef llvm_sig;
2141 gboolean virtual, calli;
2142 LLVMBuilderRef builder = *builder_ref;
2145 if (call->signature->call_convention != MONO_CALL_DEFAULT)
2146 LLVM_FAILURE (ctx, "non-default callconv");
2148 cinfo = call->cinfo;
2149 if (call->rgctx_arg_reg)
2150 cinfo->rgctx_arg = TRUE;
2151 if (call->imt_arg_reg)
2152 cinfo->imt_arg = TRUE;
2154 vretaddr = cinfo && cinfo->ret.storage == LLVMArgVtypeRetAddr;
2156 llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo, &sinfo);
2157 CHECK_FAILURE (ctx);
2159 virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE || ins->opcode == OP_RCALL_MEMBASE);
2160 calli = (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG || ins->opcode == OP_RCALL_REG);
2162 /* FIXME: Avoid creating duplicate methods */
2164 if (ins->flags & MONO_INST_HAS_METHOD) {
2168 if (cfg->compile_aot) {
2169 callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method);
2171 LLVM_FAILURE (ctx, "can't encode patch");
2173 callee = LLVMAddFunction (module, "", llvm_sig);
2176 mono_create_jit_trampoline_in_domain (mono_domain_get (),
2178 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, target);
2182 if (call->method && strstr (call->method->klass->name, "AsyncVoidMethodBuilder"))
2183 /* LLVM miscompiles async methods */
2184 LLVM_FAILURE (ctx, "#13734");
2187 MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr);
2193 memset (&ji, 0, sizeof (ji));
2194 ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR;
2195 ji.data.target = info->name;
2197 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE);
2199 if (cfg->compile_aot) {
2200 callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name);
2202 LLVM_FAILURE (ctx, "can't encode patch");
2204 callee = LLVMAddFunction (module, "", llvm_sig);
2205 target = (gpointer)mono_icall_get_wrapper (info);
2206 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, target);
2209 if (cfg->compile_aot) {
2211 if (cfg->abs_patches) {
2212 MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr);
2214 callee = get_plt_entry (ctx, llvm_sig, abs_ji->type, abs_ji->data.target);
2216 LLVM_FAILURE (ctx, "can't encode patch");
2220 LLVM_FAILURE (ctx, "aot");
2222 callee = LLVMAddFunction (module, "", llvm_sig);
2224 if (cfg->abs_patches) {
2225 MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr);
2228 * FIXME: Some trampolines might have
2229 * their own calling convention on some platforms.
2231 #ifndef TARGET_AMD64
2232 if (abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER || abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER_V4 ||
2233 abs_ji->type == MONO_PATCH_INFO_MONITOR_EXIT || abs_ji->type == MONO_PATCH_INFO_GENERIC_CLASS_INIT)
2234 LLVM_FAILURE (ctx, "trampoline with own cconv");
2236 target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE);
2237 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, target);
2241 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, (gpointer)call->fptr);
2247 int size = sizeof (gpointer);
2250 g_assert (ins->inst_offset % size == 0);
2251 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
2253 callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0));
2255 callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0));
2257 if (ins->flags & MONO_INST_HAS_METHOD) {
2262 * Collect and convert arguments
2264 nargs = (sig->param_count * 2) + sig->hasthis + vretaddr + call->rgctx_reg + call->imt_arg_reg;
2265 len = sizeof (LLVMValueRef) * nargs;
2266 args = alloca (len);
2267 memset (args, 0, len);
2268 l = call->out_ireg_args;
2270 if (call->rgctx_arg_reg) {
2271 g_assert (values [call->rgctx_arg_reg]);
2272 g_assert (sinfo.rgctx_arg_pindex < nargs);
2274 * On ARM, the imt/rgctx argument is passed in a caller save register, but some of our trampolines etc. clobber it, leading to
2275 * problems is LLVM moves the arg assignment earlier. To work around this, save the argument into a stack slot and load
2276 * it using a volatile load.
2279 if (!ctx->imt_rgctx_loc)
2280 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->lmodule->ptr_type, sizeof (gpointer));
2281 LLVMBuildStore (builder, convert (ctx, ctx->values [call->rgctx_arg_reg], ctx->lmodule->ptr_type), ctx->imt_rgctx_loc);
2282 args [sinfo.rgctx_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
2284 args [sinfo.rgctx_arg_pindex] = convert (ctx, values [call->rgctx_arg_reg], ctx->lmodule->ptr_type);
2287 if (call->imt_arg_reg) {
2288 g_assert (values [call->imt_arg_reg]);
2289 g_assert (sinfo.imt_arg_pindex < nargs);
2291 if (!ctx->imt_rgctx_loc)
2292 ctx->imt_rgctx_loc = build_alloca_llvm_type (ctx, ctx->lmodule->ptr_type, sizeof (gpointer));
2293 LLVMBuildStore (builder, convert (ctx, ctx->values [call->imt_arg_reg], ctx->lmodule->ptr_type), ctx->imt_rgctx_loc);
2294 args [sinfo.imt_arg_pindex] = mono_llvm_build_load (builder, ctx->imt_rgctx_loc, "", TRUE, LLVM_BARRIER_NONE);
2296 args [sinfo.imt_arg_pindex] = convert (ctx, values [call->imt_arg_reg], ctx->lmodule->ptr_type);
2301 if (!addresses [call->inst.dreg])
2302 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2303 g_assert (sinfo.vret_arg_pindex < nargs);
2304 args [sinfo.vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
2307 for (i = 0; i < sig->param_count + sig->hasthis; ++i) {
2310 LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL;
2314 pindex = sinfo.this_arg_pindex;
2316 pindex = sinfo.pindexes [i - 1];
2318 pindex = sinfo.pindexes [i];
2321 regpair = (guint32)(gssize)(l->data);
2322 reg = regpair & 0xffffff;
2323 args [pindex] = values [reg];
2324 switch (ainfo->storage) {
2325 case LLVMArgVtypeInReg:
2326 case LLVMArgAsFpArgs: {
2329 g_assert (addresses [reg]);
2330 emit_vtype_to_args (ctx, builder, sig->params [i - sig->hasthis], addresses [reg], ainfo, args + pindex, &nargs);
2334 // FIXME: Get rid of the VMOVE
2337 case LLVMArgVtypeByVal:
2338 g_assert (addresses [reg]);
2339 args [pindex] = addresses [reg];
2341 case LLVMArgAsIArgs:
2342 g_assert (addresses [reg]);
2343 args [pindex] = LLVMBuildLoad (ctx->builder, convert (ctx, addresses [reg], LLVMPointerType (LLVMArrayType (IntPtrType (), ainfo->nslots), 0)), "");
2346 g_assert (args [pindex]);
2347 if (i == 0 && sig->hasthis)
2348 args [pindex] = convert (ctx, args [pindex], ThisType ());
2350 args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, sig->params [i - sig->hasthis]));
2357 // FIXME: Align call sites
2363 lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig));
2366 * Modify cconv and parameter attributes to pass rgctx/imt correctly.
2368 #if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG)
2369 g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG);
2371 /* The two can't be used together, so use only one LLVM calling conv to pass them */
2372 g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg));
2374 LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv);
2376 if (call->rgctx_arg_reg)
2377 LLVMAddInstrAttribute (lcall, 1 + sinfo.rgctx_arg_pindex, LLVMInRegAttribute);
2378 if (call->imt_arg_reg)
2379 LLVMAddInstrAttribute (lcall, 1 + sinfo.imt_arg_pindex, LLVMInRegAttribute);
2381 /* Add byval attributes if needed */
2382 for (i = 0; i < sig->param_count; ++i) {
2383 LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i + sig->hasthis] : NULL;
2385 if (ainfo && ainfo->storage == LLVMArgVtypeByVal) {
2386 LLVMAddInstrAttribute (lcall, 1 + sinfo.pindexes [i], LLVMByValAttribute);
2391 * Convert the result
2394 switch (cinfo->ret.storage) {
2395 case LLVMArgVtypeInReg: {
2396 LLVMValueRef regs [2];
2398 if (!addresses [ins->dreg])
2399 addresses [ins->dreg] = build_alloca (ctx, sig->ret);
2401 regs [0] = LLVMBuildExtractValue (builder, lcall, 0, "");
2402 if (cinfo->ret.pair_storage [1] != LLVMArgNone)
2403 regs [1] = LLVMBuildExtractValue (builder, lcall, 1, "");
2404 emit_args_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs);
2407 case LLVMArgVtypeByVal:
2408 if (!addresses [call->inst.dreg])
2409 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2410 LLVMBuildStore (builder, lcall, addresses [call->inst.dreg]);
2412 case LLVMArgFpStruct:
2413 if (!addresses [call->inst.dreg])
2414 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2415 LLVMBuildStore (builder, lcall, convert_full (ctx, addresses [call->inst.dreg], LLVMPointerType (LLVMTypeOf (lcall), 0), FALSE));
2418 if (sig->ret->type != MONO_TYPE_VOID && !vretaddr)
2419 /* If the method returns an unsigned value, need to zext it */
2420 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
2424 if (sig->ret->type != MONO_TYPE_VOID && !vretaddr)
2425 /* If the method returns an unsigned value, need to zext it */
2426 values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret));
2430 if (!addresses [call->inst.dreg])
2431 addresses [call->inst.dreg] = build_alloca (ctx, sig->ret);
2432 g_assert (sinfo.vret_arg_pindex < nargs);
2433 args [sinfo.vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), "");
2436 *builder_ref = ctx->builder;
2438 g_free (sinfo.pindexes);
2446 process_bb (EmitContext *ctx, MonoBasicBlock *bb)
2448 MonoCompile *cfg = ctx->cfg;
2449 MonoMethodSignature *sig = ctx->sig;
2450 LLVMValueRef method = ctx->lmethod;
2451 LLVMValueRef *values = ctx->values;
2452 LLVMValueRef *addresses = ctx->addresses;
2453 LLVMCallInfo *linfo = ctx->linfo;
2454 LLVMModuleRef module = ctx->module;
2455 BBInfo *bblocks = ctx->bblocks;
2457 LLVMBasicBlockRef cbb;
2458 LLVMBuilderRef builder, starting_builder;
2459 gboolean has_terminator;
2461 LLVMValueRef lhs, rhs;
2464 cbb = get_bb (ctx, bb);
2465 builder = create_builder (ctx);
2466 ctx->builder = builder;
2467 LLVMPositionBuilderAtEnd (builder, cbb);
2469 if (bb == cfg->bb_entry)
2470 emit_entry_bb (ctx, builder);
2471 CHECK_FAILURE (ctx);
2473 if (bb->flags & BB_EXCEPTION_HANDLER) {
2475 LLVMValueRef personality;
2476 LLVMBasicBlockRef target_bb;
2478 static gint32 mapping_inited;
2479 static int ti_generator;
2482 LLVMValueRef type_info;
2485 if (!bblocks [bb->block_num].invoke_target) {
2487 * LLVM asserts if llvm.eh.selector is called from a bblock which
2488 * doesn't have an invoke pointing at it.
2489 * Update: LLVM no longer asserts, but some tests in exceptions.exe now fail.
2491 LLVM_FAILURE (ctx, "handler without invokes");
2494 // <resultval> = landingpad <somety> personality <type> <pers_fn> <clause>+
2496 if (cfg->compile_aot) {
2497 /* Use a dummy personality function */
2498 personality = LLVMGetNamedFunction (module, "mono_aot_personality");
2499 g_assert (personality);
2501 personality = LLVMGetNamedFunction (module, "mono_personality");
2502 if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0)
2503 LLVMAddGlobalMapping (ctx->lmodule->ee, personality, mono_personality);
2506 i8ptr = LLVMPointerType (LLVMInt8Type (), 0);
2508 clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1;
2511 * Create the type info
2513 sprintf (ti_name, "type_info_%d", ti_generator);
2516 if (cfg->compile_aot) {
2517 /* decode_eh_frame () in aot-runtime.c will decode this */
2518 type_info = LLVMAddGlobal (module, LLVMInt32Type (), ti_name);
2519 LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE));
2522 * These symbols are not really used, the clause_index is embedded into the EH tables generated by DwarfMonoException in LLVM.
2524 LLVMSetLinkage (type_info, LLVMInternalLinkage);
2527 * Enabling this causes llc to crash:
2528 * http://llvm.org/bugs/show_bug.cgi?id=6102
2530 //LLVM_FAILURE (ctx, "aot+clauses");
2532 // test_0_invalid_unbox_arrays () fails
2533 LLVM_FAILURE (ctx, "aot+clauses");
2537 * After the cfg mempool is freed, the type info will point to stale memory,
2538 * but this is not a problem, since we decode it once in exception_cb during
2541 ti = mono_mempool_alloc (cfg->mempool, sizeof (gint32));
2542 *(gint32*)ti = clause_index;
2544 type_info = LLVMAddGlobal (module, i8ptr, ti_name);
2546 LLVMAddGlobalMapping (ctx->lmodule->ee, type_info, ti);
2550 LLVMTypeRef members [2], ret_type;
2551 LLVMValueRef landing_pad;
2553 members [0] = i8ptr;
2554 members [1] = LLVMInt32Type ();
2555 ret_type = LLVMStructType (members, 2, FALSE);
2557 landing_pad = LLVMBuildLandingPad (builder, ret_type, personality, 1, "");
2558 LLVMAddClause (landing_pad, type_info);
2560 /* Store the exception into the exvar */
2561 if (bb->in_scount == 1) {
2562 g_assert (bb->in_scount == 1);
2563 exvar = bb->in_stack [0];
2565 // FIXME: This is shared with filter clauses ?
2566 g_assert (!values [exvar->dreg]);
2568 values [exvar->dreg] = LLVMBuildExtractValue (builder, landing_pad, 0, "ex_obj");
2569 emit_volatile_store (ctx, exvar->dreg);
2573 /* Start a new bblock which CALL_HANDLER can branch to */
2574 target_bb = bblocks [bb->block_num].call_handler_target_bb;
2576 LLVMBuildBr (builder, target_bb);
2578 ctx->builder = builder = create_builder (ctx);
2579 LLVMPositionBuilderAtEnd (ctx->builder, target_bb);
2581 ctx->bblocks [bb->block_num].end_bblock = target_bb;
2585 has_terminator = FALSE;
2586 starting_builder = builder;
2587 for (ins = bb->code; ins; ins = ins->next) {
2588 const char *spec = LLVM_INS_INFO (ins->opcode);
2590 char dname_buf [128];
2592 emit_dbg_loc (ctx, builder, ins->cil_code);
2595 if (nins > 5000 && builder == starting_builder) {
2596 /* some steps in llc are non-linear in the size of basic blocks, see #5714 */
2597 LLVM_FAILURE (ctx, "basic block too long");
2601 /* There could be instructions after a terminator, skip them */
2604 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) {
2605 sprintf (dname_buf, "t%d", ins->dreg);
2609 if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') {
2610 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1);
2612 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
2613 lhs = emit_volatile_load (ctx, ins->sreg1);
2615 /* It is ok for SETRET to have an uninitialized argument */
2616 if (!values [ins->sreg1] && ins->opcode != OP_SETRET)
2617 LLVM_FAILURE (ctx, "sreg1");
2618 lhs = values [ins->sreg1];
2624 if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') {
2625 MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2);
2626 if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) {
2627 rhs = emit_volatile_load (ctx, ins->sreg2);
2629 if (!values [ins->sreg2])
2630 LLVM_FAILURE (ctx, "sreg2");
2631 rhs = values [ins->sreg2];
2637 //mono_print_ins (ins);
2638 switch (ins->opcode) {
2641 case OP_LIVERANGE_START:
2642 case OP_LIVERANGE_END:
2645 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE);
2648 #if SIZEOF_VOID_P == 4
2649 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
2651 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE);
2655 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0);
2659 values [ins->dreg] = LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0);
2661 values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ());
2663 case OP_DUMMY_ICONST:
2664 values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
2666 case OP_DUMMY_I8CONST:
2667 values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
2669 case OP_DUMMY_R8CONST:
2670 values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), 0.0f);
2673 LLVMBuildBr (builder, get_bb (ctx, ins->inst_target_bb));
2674 has_terminator = TRUE;
2680 LLVMBasicBlockRef new_bb;
2681 LLVMBuilderRef new_builder;
2683 // The default branch is already handled
2684 // FIXME: Handle it here
2686 /* Start new bblock */
2687 sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++);
2688 new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name);
2690 lhs = convert (ctx, lhs, LLVMInt32Type ());
2691 v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass));
2692 for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) {
2693 MonoBasicBlock *target_bb = ins->inst_many_bb [i];
2695 LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb));
2698 new_builder = create_builder (ctx);
2699 LLVMPositionBuilderAtEnd (new_builder, new_bb);
2700 LLVMBuildUnreachable (new_builder);
2702 has_terminator = TRUE;
2703 g_assert (!ins->next);
2709 if (linfo->ret.storage == LLVMArgVtypeInReg) {
2710 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
2711 LLVMValueRef part1, retval;
2714 size = get_vtype_size (sig->ret);
2716 g_assert (addresses [ins->sreg1]);
2718 g_assert (linfo->ret.pair_storage [0] == LLVMArgInIReg);
2719 g_assert (linfo->ret.pair_storage [1] == LLVMArgNone);
2721 part1 = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMIntType (size * 8), 0), ""), ""), IntPtrType ());
2723 retval = LLVMBuildInsertValue (builder, LLVMGetUndef (ret_type), part1, 0, "");
2725 LLVMBuildRet (builder, retval);
2729 if (linfo->ret.storage == LLVMArgVtypeByVal) {
2730 LLVMValueRef retval;
2732 g_assert (addresses [ins->sreg1]);
2733 retval = LLVMBuildLoad (builder, addresses [ins->sreg1], "");
2734 LLVMBuildRet (builder, retval);
2738 if (linfo->ret.storage == LLVMArgVtypeRetAddr) {
2739 LLVMBuildRetVoid (builder);
2743 if (linfo->ret.storage == LLVMArgFpStruct) {
2744 LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method)));
2745 LLVMValueRef retval;
2747 g_assert (addresses [ins->sreg1]);
2748 retval = LLVMBuildLoad (builder, convert (ctx, addresses [ins->sreg1], LLVMPointerType (ret_type, 0)), "");
2749 LLVMBuildRet (builder, retval);
2753 if (!lhs || ctx->is_dead [ins->sreg1]) {
2755 * The method did not set its return value, probably because it
2756 * ends with a throw.
2759 LLVMBuildRetVoid (builder);
2761 LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret)));
2763 LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret)));
2765 has_terminator = TRUE;
2772 case OP_ICOMPARE_IMM:
2773 case OP_LCOMPARE_IMM:
2774 case OP_COMPARE_IMM: {
2778 if (ins->next->opcode == OP_NOP)
2781 if (ins->next->opcode == OP_BR)
2782 /* The comparison result is not needed */
2785 rel = mono_opcode_to_cond (ins->next->opcode);
2787 if (ins->opcode == OP_ICOMPARE_IMM) {
2788 lhs = convert (ctx, lhs, LLVMInt32Type ());
2789 rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
2791 if (ins->opcode == OP_LCOMPARE_IMM) {
2792 lhs = convert (ctx, lhs, LLVMInt64Type ());
2793 rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
2795 if (ins->opcode == OP_LCOMPARE) {
2796 lhs = convert (ctx, lhs, LLVMInt64Type ());
2797 rhs = convert (ctx, rhs, LLVMInt64Type ());
2799 if (ins->opcode == OP_ICOMPARE) {
2800 lhs = convert (ctx, lhs, LLVMInt32Type ());
2801 rhs = convert (ctx, rhs, LLVMInt32Type ());
2805 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
2806 rhs = convert (ctx, rhs, LLVMTypeOf (lhs));
2807 else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind)
2808 lhs = convert (ctx, lhs, LLVMTypeOf (rhs));
2811 /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */
2812 if (ins->opcode == OP_FCOMPARE) {
2813 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
2814 } else if (ins->opcode == OP_RCOMPARE) {
2815 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
2816 } else if (ins->opcode == OP_COMPARE_IMM) {
2817 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && ins->inst_imm == 0)
2818 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, LLVMConstNull (LLVMTypeOf (lhs)), "");
2820 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), "");
2821 } else if (ins->opcode == OP_LCOMPARE_IMM) {
2822 if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) {
2823 /* The immediate is encoded in two fields */
2824 guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm);
2825 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), "");
2827 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), "");
2830 else if (ins->opcode == OP_COMPARE) {
2831 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind && LLVMTypeOf (lhs) == LLVMTypeOf (rhs))
2832 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
2834 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), "");
2836 cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, "");
2838 if (MONO_IS_COND_BRANCH_OP (ins->next)) {
2839 if (ins->next->inst_true_bb == ins->next->inst_false_bb) {
2841 * If the target bb contains PHI instructions, LLVM requires
2842 * two PHI entries for this bblock, while we only generate one.
2843 * So convert this to an unconditional bblock. (bxc #171).
2845 LLVMBuildBr (builder, get_bb (ctx, ins->next->inst_true_bb));
2847 LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb));
2849 has_terminator = TRUE;
2850 } else if (MONO_IS_SETCC (ins->next)) {
2851 sprintf (dname_buf, "t%d", ins->next->dreg);
2853 values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
2855 /* Add stores for volatile variables */
2856 emit_volatile_store (ctx, ins->next->dreg);
2857 } else if (MONO_IS_COND_EXC (ins->next)) {
2858 emit_cond_system_exception (ctx, bb, ins->next->inst_p1, cmp);
2859 CHECK_FAILURE (ctx);
2860 builder = ctx->builder;
2862 LLVM_FAILURE (ctx, "next");
2876 rel = mono_opcode_to_cond (ins->opcode);
2878 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), "");
2879 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
2890 rel = mono_opcode_to_cond (ins->opcode);
2892 cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMFloatType ()), convert (ctx, rhs, LLVMFloatType ()), "");
2893 values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname);
2901 gboolean empty = TRUE;
2903 /* Check that all input bblocks really branch to us */
2904 for (i = 0; i < bb->in_count; ++i) {
2905 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED)
2906 ins->inst_phi_args [i + 1] = -1;
2912 /* LLVM doesn't like phi instructions with zero operands */
2913 ctx->is_dead [ins->dreg] = TRUE;
2917 /* Created earlier, insert it now */
2918 LLVMInsertIntoBuilder (builder, values [ins->dreg]);
2920 for (i = 0; i < ins->inst_phi_args [0]; i++) {
2921 int sreg1 = ins->inst_phi_args [i + 1];
2925 * Count the number of times the incoming bblock branches to us,
2926 * since llvm requires a separate entry for each.
2928 if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) {
2929 MonoInst *switch_ins = bb->in_bb [i]->last_ins;
2932 for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) {
2933 if (switch_ins->inst_many_bb [j] == bb)
2940 /* Remember for later */
2941 for (j = 0; j < count; ++j) {
2942 PhiNode *node = mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode));
2945 node->in_bb = bb->in_bb [i];
2947 bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node);
2957 values [ins->dreg] = lhs;
2961 MonoInst *var = get_vreg_to_inst (cfg, ins->dreg);
2964 values [ins->dreg] = lhs;
2966 if (var && var->klass->byval_arg.type == MONO_TYPE_R4) {
2968 * This is added by the spilling pass in case of the JIT,
2969 * but we have to do it ourselves.
2971 values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ());
2975 case OP_MOVE_F_TO_I4: {
2976 values [ins->dreg] = LLVMBuildBitCast (builder, LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""), LLVMInt32Type (), "");
2979 case OP_MOVE_I4_TO_F: {
2980 values [ins->dreg] = LLVMBuildFPExt (builder, LLVMBuildBitCast (builder, lhs, LLVMFloatType (), ""), LLVMDoubleType (), "");
2983 case OP_MOVE_F_TO_I8: {
2984 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMInt64Type (), "");
2987 case OP_MOVE_I8_TO_F: {
2988 values [ins->dreg] = LLVMBuildBitCast (builder, lhs, LLVMDoubleType (), "");
3021 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
3022 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
3024 #ifdef MONO_ARCH_NEED_DIV_CHECK
3025 switch (ins->opcode) {
3036 cmp = LLVMBuildICmp (builder, LLVMIntEQ, rhs, LLVMConstInt (LLVMTypeOf (rhs), 0, FALSE), "");
3037 emit_cond_system_exception (ctx, bb, "DivideByZeroException", cmp);
3038 CHECK_FAILURE (ctx);
3039 builder = ctx->builder;
3047 switch (ins->opcode) {
3050 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname);
3054 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname);
3058 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname);
3062 values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname);
3066 values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname);
3070 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname);
3074 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname);
3078 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
3082 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname);
3086 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname);
3090 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname);
3094 values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname);
3098 values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname);
3102 values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname);
3106 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
3109 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
3112 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
3116 g_assert_not_reached ();
3123 lhs = convert (ctx, lhs, LLVMFloatType ());
3124 rhs = convert (ctx, rhs, LLVMFloatType ());
3125 switch (ins->opcode) {
3127 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname);
3130 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname);
3133 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname);
3136 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname);
3139 g_assert_not_reached ();
3148 case OP_IREM_UN_IMM:
3150 case OP_IDIV_UN_IMM:
3156 case OP_ISHR_UN_IMM:
3165 case OP_LSHR_UN_IMM:
3171 case OP_SHR_UN_IMM: {
3174 if (spec [MONO_INST_SRC1] == 'l') {
3175 imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE);
3177 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
3180 #if SIZEOF_VOID_P == 4
3181 if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM)
3182 imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
3185 if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind)
3186 lhs = convert (ctx, lhs, IntPtrType ());
3187 imm = convert (ctx, imm, LLVMTypeOf (lhs));
3188 switch (ins->opcode) {
3192 values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname);
3196 values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname);
3200 values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname);
3204 values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname);
3206 case OP_IDIV_UN_IMM:
3207 case OP_LDIV_UN_IMM:
3208 values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname);
3212 values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname);
3214 case OP_IREM_UN_IMM:
3215 values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname);
3220 values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname);
3224 values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname);
3228 values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname);
3233 values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname);
3238 values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname);
3240 case OP_ISHR_UN_IMM:
3241 /* This is used to implement conv.u4, so the lhs could be an i8 */
3242 lhs = convert (ctx, lhs, LLVMInt32Type ());
3243 imm = convert (ctx, imm, LLVMInt32Type ());
3244 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
3246 case OP_LSHR_UN_IMM:
3248 values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname);
3251 g_assert_not_reached ();
3256 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
3259 values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname);
3262 lhs = convert (ctx, lhs, LLVMDoubleType ());
3263 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname);
3266 lhs = convert (ctx, lhs, LLVMFloatType ());
3267 values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMFloatType (), 0.0), lhs, dname);
3270 guint32 v = 0xffffffff;
3271 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname);
3275 guint64 v = 0xffffffffffffffffLL;
3276 values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname);
3279 #if defined(TARGET_X86) || defined(TARGET_AMD64)
3281 LLVMValueRef v1, v2;
3283 v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), "");
3284 v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, "");
3285 values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname);
3290 case OP_ICONV_TO_I1:
3291 case OP_ICONV_TO_I2:
3292 case OP_ICONV_TO_I4:
3293 case OP_ICONV_TO_U1:
3294 case OP_ICONV_TO_U2:
3295 case OP_ICONV_TO_U4:
3296 case OP_LCONV_TO_I1:
3297 case OP_LCONV_TO_I2:
3298 case OP_LCONV_TO_U1:
3299 case OP_LCONV_TO_U2:
3300 case OP_LCONV_TO_U4: {
3303 sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2);
3305 /* Have to do two casts since our vregs have type int */
3306 v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), "");
3308 values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname);
3310 values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname);
3313 case OP_ICONV_TO_I8:
3314 values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname);
3316 case OP_ICONV_TO_U8:
3317 values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname);
3319 case OP_FCONV_TO_I4:
3320 case OP_RCONV_TO_I4:
3321 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname);
3323 case OP_FCONV_TO_I1:
3324 case OP_RCONV_TO_I1:
3325 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
3327 case OP_FCONV_TO_U1:
3328 case OP_RCONV_TO_U1:
3329 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), "");
3331 case OP_FCONV_TO_I2:
3332 case OP_RCONV_TO_I2:
3333 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
3335 case OP_FCONV_TO_U2:
3336 case OP_RCONV_TO_U2:
3337 values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), "");
3339 case OP_FCONV_TO_I8:
3340 case OP_RCONV_TO_I8:
3341 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname);
3344 values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname);
3346 case OP_ICONV_TO_R8:
3347 case OP_LCONV_TO_R8:
3348 values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname);
3350 case OP_LCONV_TO_R_UN:
3351 values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname);
3353 #if SIZEOF_VOID_P == 4
3356 case OP_LCONV_TO_I4:
3357 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
3359 case OP_ICONV_TO_R4:
3360 case OP_LCONV_TO_R4:
3361 v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), "");
3363 values [ins->dreg] = v;
3365 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
3367 case OP_FCONV_TO_R4:
3368 v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), "");
3370 values [ins->dreg] = v;
3372 values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname);
3374 case OP_RCONV_TO_R8:
3375 values [ins->dreg] = LLVMBuildFPExt (builder, lhs, LLVMDoubleType (), dname);
3377 case OP_RCONV_TO_R4:
3378 values [ins->dreg] = lhs;
3381 values [ins->dreg] = LLVMBuildSExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
3384 values [ins->dreg] = LLVMBuildZExt (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMInt64Type (), dname);
3387 values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname);
3389 case OP_LOCALLOC_IMM: {
3392 guint32 size = ins->inst_imm;
3393 size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1);
3395 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, "");
3397 if (ins->flags & MONO_INST_INIT) {
3398 LLVMValueRef args [5];
3401 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3402 args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE);
3403 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
3404 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3405 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, "");
3408 values [ins->dreg] = v;
3412 LLVMValueRef v, size;
3414 size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), "");
3416 v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, "");
3418 if (ins->flags & MONO_INST_INIT) {
3419 LLVMValueRef args [5];
3422 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3424 args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE);
3425 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3426 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, "");
3428 values [ins->dreg] = v;
3432 case OP_LOADI1_MEMBASE:
3433 case OP_LOADU1_MEMBASE:
3434 case OP_LOADI2_MEMBASE:
3435 case OP_LOADU2_MEMBASE:
3436 case OP_LOADI4_MEMBASE:
3437 case OP_LOADU4_MEMBASE:
3438 case OP_LOADI8_MEMBASE:
3439 case OP_LOADR4_MEMBASE:
3440 case OP_LOADR8_MEMBASE:
3441 case OP_LOAD_MEMBASE:
3449 LLVMValueRef base, index, addr;
3451 gboolean sext = FALSE, zext = FALSE;
3452 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3454 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3459 if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) {
3460 addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE);
3465 if (ins->inst_offset == 0) {
3467 } else if (ins->inst_offset % size != 0) {
3468 /* Unaligned load */
3469 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
3470 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
3472 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3473 addr = LLVMBuildGEP (builder, convert (ctx, base, LLVMPointerType (t, 0)), &index, 1, "");
3477 addr = convert (ctx, addr, LLVMPointerType (t, 0));
3479 values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, dname, is_volatile);
3481 if (!is_volatile && (ins->flags & MONO_INST_INVARIANT_LOAD)) {
3483 * These will signal LLVM that these loads do not alias any stores, and
3484 * they can't fail, allowing them to be hoisted out of loops.
3486 set_invariant_load_flag (values [ins->dreg]);
3487 set_metadata_flag (values [ins->dreg], "mono.nofail.load");
3491 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3493 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3494 else if (!cfg->r4fp && ins->opcode == OP_LOADR4_MEMBASE)
3495 values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname);
3499 case OP_STOREI1_MEMBASE_REG:
3500 case OP_STOREI2_MEMBASE_REG:
3501 case OP_STOREI4_MEMBASE_REG:
3502 case OP_STOREI8_MEMBASE_REG:
3503 case OP_STORER4_MEMBASE_REG:
3504 case OP_STORER8_MEMBASE_REG:
3505 case OP_STORE_MEMBASE_REG: {
3507 LLVMValueRef index, addr;
3509 gboolean sext = FALSE, zext = FALSE;
3510 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3512 if (!values [ins->inst_destbasereg])
3513 LLVM_FAILURE (ctx, "inst_destbasereg");
3515 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3517 if (ins->inst_offset % size != 0) {
3518 /* Unaligned store */
3519 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
3520 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
3522 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3523 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
3525 emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
3529 case OP_STOREI1_MEMBASE_IMM:
3530 case OP_STOREI2_MEMBASE_IMM:
3531 case OP_STOREI4_MEMBASE_IMM:
3532 case OP_STOREI8_MEMBASE_IMM:
3533 case OP_STORE_MEMBASE_IMM: {
3535 LLVMValueRef index, addr;
3537 gboolean sext = FALSE, zext = FALSE;
3538 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3540 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3542 if (ins->inst_offset % size != 0) {
3543 /* Unaligned store */
3544 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE);
3545 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, "");
3547 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3548 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
3550 emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile);
3555 emit_load (ctx, bb, &builder, sizeof (gpointer), convert (ctx, lhs, LLVMPointerType (IntPtrType (), 0)), "", TRUE);
3557 case OP_OUTARG_VTRETADDR:
3565 case OP_VOIDCALL_MEMBASE:
3566 case OP_CALL_MEMBASE:
3567 case OP_LCALL_MEMBASE:
3568 case OP_FCALL_MEMBASE:
3569 case OP_RCALL_MEMBASE:
3570 case OP_VCALL_MEMBASE:
3571 case OP_VOIDCALL_REG:
3576 case OP_VCALL_REG: {
3577 process_call (ctx, bb, &builder, ins);
3578 CHECK_FAILURE (ctx);
3583 LLVMValueRef indexes [2];
3585 LLVMValueRef got_entry_addr;
3588 * FIXME: Can't allocate from the cfg mempool since that is freed if
3589 * the LLVM compile fails.
3591 ji = g_new0 (MonoJumpInfo, 1);
3592 ji->type = (MonoJumpInfoType)ins->inst_i1;
3593 ji->data.target = ins->inst_p0;
3595 ji = mono_aot_patch_info_dup (ji);
3597 ji->next = cfg->patch_info;
3598 cfg->patch_info = ji;
3600 //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0);
3601 got_offset = mono_aot_get_got_offset (cfg->patch_info);
3602 ctx->lmodule->max_got_offset = MAX (ctx->lmodule->max_got_offset, got_offset);
3604 indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3605 indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE);
3606 got_entry_addr = LLVMBuildGEP (builder, ctx->lmodule->got_var, indexes, 2, "");
3608 values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, dname);
3609 set_invariant_load_flag (values [ins->dreg]);
3612 case OP_NOT_REACHED:
3613 LLVMBuildUnreachable (builder);
3614 has_terminator = TRUE;
3615 g_assert (bb->block_num < cfg->max_block_num);
3616 ctx->unreachable [bb->block_num] = TRUE;
3617 /* Might have instructions after this */
3619 MonoInst *next = ins->next;
3621 * FIXME: If later code uses the regs defined by these instructions,
3622 * compilation will fail.
3624 MONO_DELETE_INS (bb, next);
3628 MonoInst *var = ins->inst_p0;
3630 values [ins->dreg] = addresses [var->dreg];
3634 LLVMValueRef args [1];
3636 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3637 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sin.f64"), args, 1, dname);
3641 LLVMValueRef args [1];
3643 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3644 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.cos.f64"), args, 1, dname);
3648 LLVMValueRef args [1];
3651 /* This no longer seems to happen */
3653 * LLVM optimizes sqrt(nan) into undefined in
3654 * lib/Analysis/ConstantFolding.cpp
3655 * Also, sqrt(NegativeInfinity) is optimized into 0.
3657 LLVM_FAILURE (ctx, "sqrt");
3659 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3660 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sqrt.f64"), args, 1, dname);
3664 LLVMValueRef args [1];
3666 args [0] = convert (ctx, lhs, LLVMDoubleType ());
3667 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "fabs"), args, 1, dname);
3681 lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
3682 rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST]));
3684 switch (ins->opcode) {
3687 v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, "");
3691 v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, "");
3695 v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, "");
3699 v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, "");
3702 g_assert_not_reached ();
3705 values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname);
3708 case OP_ATOMIC_EXCHANGE_I4:
3709 case OP_ATOMIC_EXCHANGE_I8: {
3710 LLVMValueRef args [2];
3713 if (ins->opcode == OP_ATOMIC_EXCHANGE_I4)
3714 t = LLVMInt32Type ();
3716 t = LLVMInt64Type ();
3718 g_assert (ins->inst_offset == 0);
3720 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
3721 args [1] = convert (ctx, rhs, t);
3723 values [ins->dreg] = mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_XCHG, args [0], args [1]);
3726 case OP_ATOMIC_ADD_I4:
3727 case OP_ATOMIC_ADD_I8: {
3728 LLVMValueRef args [2];
3731 if (ins->opcode == OP_ATOMIC_ADD_I4)
3732 t = LLVMInt32Type ();
3734 t = LLVMInt64Type ();
3736 g_assert (ins->inst_offset == 0);
3738 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
3739 args [1] = convert (ctx, rhs, t);
3740 values [ins->dreg] = LLVMBuildAdd (builder, mono_llvm_build_atomic_rmw (builder, LLVM_ATOMICRMW_OP_ADD, args [0], args [1]), args [1], dname);
3743 case OP_ATOMIC_CAS_I4:
3744 case OP_ATOMIC_CAS_I8: {
3745 LLVMValueRef args [3], val;
3748 if (ins->opcode == OP_ATOMIC_CAS_I4)
3749 t = LLVMInt32Type ();
3751 t = LLVMInt64Type ();
3753 args [0] = convert (ctx, lhs, LLVMPointerType (t, 0));
3755 args [1] = convert (ctx, values [ins->sreg3], t);
3757 args [2] = convert (ctx, values [ins->sreg2], t);
3758 val = mono_llvm_build_cmpxchg (builder, args [0], args [1], args [2]);
3759 #if LLVM_API_VERSION >= 1
3760 /* cmpxchg returns a pair */
3761 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, "");
3763 values [ins->dreg] = val;
3767 case OP_MEMORY_BARRIER: {
3768 mono_llvm_build_fence (builder, (BarrierKind) ins->backend.memory_barrier_kind);
3771 case OP_ATOMIC_LOAD_I1:
3772 case OP_ATOMIC_LOAD_I2:
3773 case OP_ATOMIC_LOAD_I4:
3774 case OP_ATOMIC_LOAD_I8:
3775 case OP_ATOMIC_LOAD_U1:
3776 case OP_ATOMIC_LOAD_U2:
3777 case OP_ATOMIC_LOAD_U4:
3778 case OP_ATOMIC_LOAD_U8:
3779 case OP_ATOMIC_LOAD_R4:
3780 case OP_ATOMIC_LOAD_R8: {
3781 LLVM_FAILURE (ctx, "atomic mono.load intrinsic");
3782 #if LLVM_API_VERSION >= 4
3784 gboolean sext, zext;
3786 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3787 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
3788 LLVMValueRef index, addr;
3790 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3795 if (ins->inst_offset != 0) {
3796 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3797 addr = LLVMBuildGEP (builder, convert (ctx, lhs, LLVMPointerType (t, 0)), &index, 1, "");
3802 addr = convert (ctx, addr, LLVMPointerType (t, 0));
3804 values [ins->dreg] = emit_load_general (ctx, bb, &builder, size, addr, dname, is_volatile, barrier);
3807 values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3809 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname);
3811 LLVM_FAILURE (ctx, "atomic mono.load intrinsic");
3815 case OP_ATOMIC_STORE_I1:
3816 case OP_ATOMIC_STORE_I2:
3817 case OP_ATOMIC_STORE_I4:
3818 case OP_ATOMIC_STORE_I8:
3819 case OP_ATOMIC_STORE_U1:
3820 case OP_ATOMIC_STORE_U2:
3821 case OP_ATOMIC_STORE_U4:
3822 case OP_ATOMIC_STORE_U8:
3823 case OP_ATOMIC_STORE_R4:
3824 case OP_ATOMIC_STORE_R8: {
3825 LLVM_FAILURE (ctx, "atomic mono.load intrinsic");
3826 #if LLVM_API_VERSION >= 4
3828 gboolean sext, zext;
3830 gboolean is_volatile = (ins->flags & MONO_INST_FAULT);
3831 BarrierKind barrier = (BarrierKind) ins->backend.memory_barrier_kind;
3832 LLVMValueRef index, addr, value;
3834 if (!values [ins->inst_destbasereg])
3835 LLVM_FAILURE (ctx, "inst_destbasereg");
3837 t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext);
3839 index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE);
3840 addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, "");
3841 value = convert (ctx, values [ins->sreg1], t);
3843 emit_store_general (ctx, bb, &builder, size, value, addr, is_volatile, barrier);
3846 LLVM_FAILURE (ctx, "atomic mono.store intrinsic");
3850 case OP_RELAXED_NOP: {
3851 #if defined(TARGET_AMD64) || defined(TARGET_X86)
3852 emit_call (ctx, bb, &builder, LLVMGetNamedFunction (ctx->module, "llvm.x86.sse2.pause"), NULL, 0);
3859 #if (defined(TARGET_AMD64) || defined(TARGET_X86)) && defined(__linux__)
3861 // 257 == FS segment register
3862 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257);
3864 // 256 == GS segment register
3865 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
3868 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), "");
3869 #elif defined(TARGET_AMD64) && defined(TARGET_OSX)
3870 /* See mono_amd64_emit_tls_get () */
3871 int offset = mono_amd64_get_tls_gs_offset () + (ins->inst_offset * 8);
3873 // 256 == GS segment register
3874 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
3875 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), offset, TRUE), ptrtype, ""), "");
3877 LLVM_FAILURE (ctx, "opcode tls-get");
3882 case OP_TLS_GET_REG: {
3883 #if defined(TARGET_AMD64) && defined(TARGET_OSX)
3884 /* See emit_tls_get_reg () */
3885 // 256 == GS segment register
3886 LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256);
3887 values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, convert (ctx, lhs, LLVMInt32Type ()), ptrtype, ""), "");
3889 LLVM_FAILURE (ctx, "opcode tls-get");
3898 case OP_IADD_OVF_UN:
3900 case OP_ISUB_OVF_UN:
3902 case OP_IMUL_OVF_UN:
3903 #if SIZEOF_VOID_P == 8
3905 case OP_LADD_OVF_UN:
3907 case OP_LSUB_OVF_UN:
3909 case OP_LMUL_OVF_UN:
3912 LLVMValueRef args [2], val, ovf, func;
3914 args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode));
3915 args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode));
3916 func = LLVMGetNamedFunction (module, ovf_op_to_intrins (ins->opcode));
3918 val = LLVMBuildCall (builder, func, args, 2, "");
3919 values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname);
3920 ovf = LLVMBuildExtractValue (builder, val, 1, "");
3921 emit_cond_system_exception (ctx, bb, "OverflowException", ovf);
3922 CHECK_FAILURE (ctx);
3923 builder = ctx->builder;
3929 * We currently model them using arrays. Promotion to local vregs is
3930 * disabled for them in mono_handle_global_vregs () in the LLVM case,
3931 * so we always have an entry in cfg->varinfo for them.
3932 * FIXME: Is this needed ?
3935 MonoClass *klass = ins->klass;
3936 LLVMValueRef args [5];
3940 LLVM_FAILURE (ctx, "!klass");
3944 if (!addresses [ins->dreg])
3945 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3946 args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
3947 args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE);
3948 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
3950 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
3951 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
3952 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, "");
3955 case OP_DUMMY_VZERO:
3958 case OP_STOREV_MEMBASE:
3959 case OP_LOADV_MEMBASE:
3961 MonoClass *klass = ins->klass;
3962 LLVMValueRef src = NULL, dst, args [5];
3963 gboolean done = FALSE;
3967 LLVM_FAILURE (ctx, "!klass");
3971 if (mini_is_gsharedvt_klass (cfg, klass)) {
3973 LLVM_FAILURE (ctx, "gsharedvt");
3977 switch (ins->opcode) {
3978 case OP_STOREV_MEMBASE:
3979 if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg &&
3980 LLVMGetInstructionOpcode (values [ins->inst_destbasereg]) != LLVMAlloca) {
3981 /* Decomposed earlier */
3982 g_assert_not_reached ();
3985 if (!addresses [ins->sreg1]) {
3987 g_assert (values [ins->sreg1]);
3988 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0));
3989 LLVMBuildStore (builder, values [ins->sreg1], dst);
3992 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
3993 dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
3996 case OP_LOADV_MEMBASE:
3997 if (!addresses [ins->dreg])
3998 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
3999 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0));
4000 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
4003 if (!addresses [ins->sreg1])
4004 addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg);
4005 if (!addresses [ins->dreg])
4006 addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg);
4007 src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), "");
4008 dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), "");
4011 g_assert_not_reached ();
4013 CHECK_FAILURE (ctx);
4020 args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE);
4021 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4023 args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4024 args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
4025 LLVMBuildCall (builder, LLVMGetNamedFunction (module, memcpy_func_name), args, memcpy_param_count, "");
4028 case OP_LLVM_OUTARG_VT:
4029 if (!addresses [ins->sreg1]) {
4030 addresses [ins->sreg1] = build_alloca (ctx, &ins->klass->byval_arg);
4031 g_assert (values [ins->sreg1]);
4032 LLVMBuildStore (builder, values [ins->sreg1], addresses [ins->sreg1]);
4034 addresses [ins->dreg] = addresses [ins->sreg1];
4040 #if defined(TARGET_X86) || defined(TARGET_AMD64)
4042 values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg));
4045 case OP_LOADX_MEMBASE: {
4046 LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg);
4049 src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
4050 values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1);
4053 case OP_STOREX_MEMBASE: {
4054 LLVMTypeRef t = LLVMTypeOf (values [ins->sreg1]);
4057 dest = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0));
4058 mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1);
4065 values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, "");
4069 values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, "");
4075 values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, "");
4079 values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, "");
4083 values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, "");
4087 values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, "");
4090 values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, "");
4093 values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, "");
4096 values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, "");
4100 values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, "");
4111 LLVMValueRef v = NULL;
4113 switch (ins->opcode) {
4118 t = LLVMVectorType (LLVMInt32Type (), 4);
4119 rt = LLVMVectorType (LLVMFloatType (), 4);
4125 t = LLVMVectorType (LLVMInt64Type (), 2);
4126 rt = LLVMVectorType (LLVMDoubleType (), 2);
4129 t = LLVMInt32Type ();
4130 rt = LLVMInt32Type ();
4131 g_assert_not_reached ();
4134 lhs = LLVMBuildBitCast (builder, lhs, t, "");
4135 rhs = LLVMBuildBitCast (builder, rhs, t, "");
4136 switch (ins->opcode) {
4139 v = LLVMBuildAnd (builder, lhs, rhs, "");
4143 v = LLVMBuildOr (builder, lhs, rhs, "");
4147 v = LLVMBuildXor (builder, lhs, rhs, "");
4151 v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), "");
4154 values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, "");
4178 case OP_PADDB_SAT_UN:
4179 case OP_PADDW_SAT_UN:
4180 case OP_PSUBB_SAT_UN:
4181 case OP_PSUBW_SAT_UN:
4189 case OP_PMULW_HIGH_UN: {
4190 LLVMValueRef args [2];
4195 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
4202 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntEQ, lhs, rhs, ""), LLVMTypeOf (lhs), "");
4206 values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildICmp (builder, LLVMIntSGT, lhs, rhs, ""), LLVMTypeOf (lhs), "");
4214 case OP_EXTRACTX_U2:
4216 case OP_EXTRACT_U1: {
4218 gboolean zext = FALSE;
4220 t = simd_op_to_llvm_type (ins->opcode);
4222 switch (ins->opcode) {
4230 case OP_EXTRACTX_U2:
4235 t = LLVMInt32Type ();
4236 g_assert_not_reached ();
4239 lhs = LLVMBuildBitCast (builder, lhs, t, "");
4240 values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), "");
4242 values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), "");
4251 case OP_EXPAND_R8: {
4252 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
4253 LLVMValueRef mask [16], v;
4256 for (i = 0; i < 16; ++i)
4257 mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
4259 v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t));
4261 values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4262 values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), "");
4267 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4270 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4273 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4276 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4279 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4282 values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname);
4293 case OP_EXTRACT_MASK:
4300 v = convert (ctx, values [ins->sreg1], simd_op_to_llvm_type (ins->opcode));
4302 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), &v, 1, dname);
4308 LLVMValueRef args [3];
4312 args [2] = LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE);
4314 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 3, dname);
4319 /* This is only used for implementing shifts by non-immediate */
4320 values [ins->dreg] = lhs;
4331 LLVMValueRef args [3];
4334 args [1] = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE);
4336 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
4347 case OP_PSHLQ_REG: {
4348 LLVMValueRef args [3];
4351 args [1] = values [ins->sreg2];
4353 values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname);
4360 case OP_PSHUFLEW_LOW:
4361 case OP_PSHUFLEW_HIGH: {
4363 LLVMValueRef v1 = NULL, v2 = NULL, mask_values [16];
4364 int i, mask_size = 0;
4365 int imask = ins->inst_c0;
4367 /* Convert the x86 shuffle mask to LLVM's */
4368 switch (ins->opcode) {
4371 mask [0] = ((imask >> 0) & 3);
4372 mask [1] = ((imask >> 2) & 3);
4373 mask [2] = ((imask >> 4) & 3) + 4;
4374 mask [3] = ((imask >> 6) & 3) + 4;
4375 v1 = values [ins->sreg1];
4376 v2 = values [ins->sreg2];
4380 mask [0] = ((imask >> 0) & 1);
4381 mask [1] = ((imask >> 1) & 1) + 2;
4382 v1 = values [ins->sreg1];
4383 v2 = values [ins->sreg2];
4385 case OP_PSHUFLEW_LOW:
4387 mask [0] = ((imask >> 0) & 3);
4388 mask [1] = ((imask >> 2) & 3);
4389 mask [2] = ((imask >> 4) & 3);
4390 mask [3] = ((imask >> 6) & 3);
4395 v1 = values [ins->sreg1];
4396 v2 = LLVMGetUndef (LLVMTypeOf (v1));
4398 case OP_PSHUFLEW_HIGH:
4404 mask [4] = 4 + ((imask >> 0) & 3);
4405 mask [5] = 4 + ((imask >> 2) & 3);
4406 mask [6] = 4 + ((imask >> 4) & 3);
4407 mask [7] = 4 + ((imask >> 6) & 3);
4408 v1 = values [ins->sreg1];
4409 v2 = LLVMGetUndef (LLVMTypeOf (v1));
4413 mask [0] = ((imask >> 0) & 3);
4414 mask [1] = ((imask >> 2) & 3);
4415 mask [2] = ((imask >> 4) & 3);
4416 mask [3] = ((imask >> 6) & 3);
4417 v1 = values [ins->sreg1];
4418 v2 = LLVMGetUndef (LLVMTypeOf (v1));
4421 g_assert_not_reached ();
4423 for (i = 0; i < mask_size; ++i)
4424 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
4426 values [ins->dreg] =
4427 LLVMBuildShuffleVector (builder, v1, v2,
4428 LLVMConstVector (mask_values, mask_size), dname);
4432 case OP_UNPACK_LOWB:
4433 case OP_UNPACK_LOWW:
4434 case OP_UNPACK_LOWD:
4435 case OP_UNPACK_LOWQ:
4436 case OP_UNPACK_LOWPS:
4437 case OP_UNPACK_LOWPD:
4438 case OP_UNPACK_HIGHB:
4439 case OP_UNPACK_HIGHW:
4440 case OP_UNPACK_HIGHD:
4441 case OP_UNPACK_HIGHQ:
4442 case OP_UNPACK_HIGHPS:
4443 case OP_UNPACK_HIGHPD: {
4445 LLVMValueRef mask_values [16];
4446 int i, mask_size = 0;
4447 gboolean low = FALSE;
4449 switch (ins->opcode) {
4450 case OP_UNPACK_LOWB:
4454 case OP_UNPACK_LOWW:
4458 case OP_UNPACK_LOWD:
4459 case OP_UNPACK_LOWPS:
4463 case OP_UNPACK_LOWQ:
4464 case OP_UNPACK_LOWPD:
4468 case OP_UNPACK_HIGHB:
4471 case OP_UNPACK_HIGHW:
4474 case OP_UNPACK_HIGHD:
4475 case OP_UNPACK_HIGHPS:
4478 case OP_UNPACK_HIGHQ:
4479 case OP_UNPACK_HIGHPD:
4483 g_assert_not_reached ();
4487 for (i = 0; i < (mask_size / 2); ++i) {
4489 mask [(i * 2) + 1] = mask_size + i;
4492 for (i = 0; i < (mask_size / 2); ++i) {
4493 mask [(i * 2)] = (mask_size / 2) + i;
4494 mask [(i * 2) + 1] = mask_size + (mask_size / 2) + i;
4498 for (i = 0; i < mask_size; ++i)
4499 mask_values [i] = LLVMConstInt (LLVMInt32Type (), mask [i], FALSE);
4501 values [ins->dreg] =
4502 LLVMBuildShuffleVector (builder, values [ins->sreg1], values [ins->sreg2],
4503 LLVMConstVector (mask_values, mask_size), dname);
4508 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
4509 LLVMValueRef v, val;
4511 v = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4512 val = LLVMConstNull (t);
4513 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4514 val = LLVMBuildInsertElement (builder, val, v, LLVMConstInt (LLVMInt32Type (), 1, FALSE), dname);
4516 values [ins->dreg] = val;
4520 case OP_DUPPS_HIGH: {
4521 LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode);
4522 LLVMValueRef v1, v2, val;
4525 if (ins->opcode == OP_DUPPS_LOW) {
4526 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4527 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
4529 v1 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
4530 v2 = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
4532 val = LLVMConstNull (t);
4533 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 0, FALSE), "");
4534 val = LLVMBuildInsertElement (builder, val, v1, LLVMConstInt (LLVMInt32Type (), 1, FALSE), "");
4535 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 2, FALSE), "");
4536 val = LLVMBuildInsertElement (builder, val, v2, LLVMConstInt (LLVMInt32Type (), 3, FALSE), "");
4538 values [ins->dreg] = val;
4548 * EXCEPTION HANDLING
4550 case OP_IMPLICIT_EXCEPTION:
4551 /* This marks a place where an implicit exception can happen */
4552 if (bb->region != -1)
4553 LLVM_FAILURE (ctx, "implicit-exception");
4557 MonoMethodSignature *throw_sig;
4558 LLVMValueRef callee, arg;
4559 gboolean rethrow = (ins->opcode == OP_RETHROW);
4560 const char *icall_name;
4562 callee = rethrow ? ctx->lmodule->rethrow : ctx->lmodule->throw;
4563 icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception";
4566 throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1);
4567 throw_sig->ret = &mono_get_void_class ()->byval_arg;
4568 throw_sig->params [0] = &mono_get_object_class ()->byval_arg;
4569 if (cfg->compile_aot) {
4570 callee = get_plt_entry (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name);
4572 callee = LLVMAddFunction (module, icall_name, sig_to_llvm_sig (ctx, throw_sig));
4576 * LLVM doesn't push the exception argument, so we need a different
4579 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline"));
4581 LLVMAddGlobalMapping (ctx->lmodule->ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name));
4585 mono_memory_barrier ();
4587 ctx->lmodule->rethrow = callee;
4589 ctx->lmodule->throw = callee;
4591 arg = convert (ctx, lhs, type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg));
4592 emit_call (ctx, bb, &builder, callee, &arg, 1);
4595 case OP_CALL_HANDLER: {
4597 * We don't 'call' handlers, but instead simply branch to them.
4598 * The code generated by ENDFINALLY will branch back to us.
4600 LLVMBasicBlockRef noex_bb;
4602 BBInfo *info = &bblocks [ins->inst_target_bb->block_num];
4604 bb_list = info->call_handler_return_bbs;
4607 * Set the indicator variable for the finally clause.
4609 lhs = info->finally_ind;
4611 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs);
4613 /* Branch to the finally clause */
4614 LLVMBuildBr (builder, info->call_handler_target_bb);
4616 noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB");
4617 info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb);
4619 builder = ctx->builder = create_builder (ctx);
4620 LLVMPositionBuilderAtEnd (ctx->builder, noex_bb);
4622 bblocks [bb->block_num].end_bblock = noex_bb;
4625 case OP_START_HANDLER: {
4628 case OP_ENDFINALLY: {
4629 LLVMBasicBlockRef resume_bb;
4630 MonoBasicBlock *handler_bb;
4631 LLVMValueRef val, switch_ins, callee;
4635 handler_bb = g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)));
4636 g_assert (handler_bb);
4637 info = &bblocks [handler_bb->block_num];
4638 lhs = info->finally_ind;
4641 bb_list = info->call_handler_return_bbs;
4643 resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB");
4645 /* Load the finally variable */
4646 val = LLVMBuildLoad (builder, lhs, "");
4648 /* Reset the variable */
4649 LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs);
4651 /* Branch to either resume_bb, or to the bblocks in bb_list */
4652 switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list));
4654 * The other targets are added at the end to handle OP_CALL_HANDLER
4655 * opcodes processed later.
4657 info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins);
4659 builder = ctx->builder = create_builder (ctx);
4660 LLVMPositionBuilderAtEnd (ctx->builder, resume_bb);
4662 if (ctx->cfg->compile_aot) {
4663 callee = get_plt_entry (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline");
4665 callee = LLVMGetNamedFunction (module, "llvm_resume_unwind_trampoline");
4667 LLVMBuildCall (builder, callee, NULL, 0, "");
4669 LLVMBuildUnreachable (builder);
4670 has_terminator = TRUE;
4676 sprintf (reason, "opcode %s", mono_inst_name (ins->opcode));
4677 LLVM_FAILURE (ctx, reason);
4682 /* Convert the value to the type required by phi nodes */
4683 if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) {
4684 if (!values [ins->dreg])
4686 values [ins->dreg] = addresses [ins->dreg];
4688 values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]);
4691 /* Add stores for volatile variables */
4692 if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins))
4693 emit_volatile_store (ctx, ins->dreg);
4696 if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0))
4697 LLVMBuildBr (builder, get_bb (ctx, bb->next_bb));
4699 if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) {
4700 emit_dbg_loc (ctx, builder, cfg->header->code + cfg->header->code_size - 1);
4701 LLVMBuildRetVoid (builder);
4704 if (bb == cfg->bb_entry)
4705 ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry));
4714 * mono_llvm_check_method_supported:
4716 * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid
4717 * compiling a method twice.
4720 mono_llvm_check_method_supported (MonoCompile *cfg)
4722 MonoMethodHeader *header = cfg->header;
4723 MonoExceptionClause *clause;
4726 if (cfg->method->save_lmf) {
4727 cfg->exception_message = g_strdup ("lmf");
4728 cfg->disable_llvm = TRUE;
4730 if (cfg->disable_llvm)
4734 for (i = 0; i < header->num_clauses; ++i) {
4735 clause = &header->clauses [i];
4737 if (i > 0 && clause->try_offset <= header->clauses [i - 1].handler_offset + header->clauses [i - 1].handler_len) {
4739 * FIXME: Some tests still fail with nested clauses.
4741 cfg->exception_message = g_strdup ("nested clauses");
4742 cfg->disable_llvm = TRUE;
4746 if (cfg->disable_llvm)
4751 if (cfg->method->dynamic) {
4752 cfg->exception_message = g_strdup ("dynamic.");
4753 cfg->disable_llvm = TRUE;
4755 if (cfg->disable_llvm)
4760 * mono_llvm_emit_method:
4762 * Emit LLVM IL from the mono IL, and compile it to native code using LLVM.
4765 mono_llvm_emit_method (MonoCompile *cfg)
4768 MonoMethodSignature *sig;
4770 LLVMTypeRef method_type;
4771 LLVMValueRef method = NULL;
4773 LLVMValueRef *values;
4774 int i, max_block_num, bb_index;
4775 gboolean last = FALSE;
4776 GPtrArray *phi_values;
4777 LLVMCallInfo *linfo;
4779 LLVMModuleRef module;
4781 GPtrArray *bblock_list;
4782 MonoMethodHeader *header;
4783 MonoExceptionClause *clause;
4787 /* The code below might acquire the loader lock, so use it for global locking */
4788 mono_loader_lock ();
4790 /* Used to communicate with the callbacks */
4791 mono_native_tls_set_value (current_cfg_tls_id, cfg);
4793 ctx = g_new0 (EmitContext, 1);
4795 ctx->mempool = cfg->mempool;
4798 * This maps vregs to the LLVM instruction defining them
4800 values = g_new0 (LLVMValueRef, cfg->next_vreg);
4802 * This maps vregs for volatile variables to the LLVM instruction defining their
4805 ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg);
4806 ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg);
4807 ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg);
4808 phi_values = g_ptr_array_sized_new (256);
4810 * This signals whenever the vreg was defined by a phi node with no input vars
4811 * (i.e. all its input bblocks end with NOT_REACHABLE).
4813 ctx->is_dead = g_new0 (gboolean, cfg->next_vreg);
4814 /* Whenever the bblock is unreachable */
4815 ctx->unreachable = g_new0 (gboolean, cfg->max_block_num);
4817 bblock_list = g_ptr_array_sized_new (256);
4819 ctx->values = values;
4820 ctx->region_to_handler = g_hash_table_new (NULL, NULL);
4822 if (cfg->compile_aot) {
4823 ctx->lmodule = &aot_module;
4824 method_name = mono_aot_get_method_name (cfg);
4825 cfg->llvm_method_name = g_strdup (method_name);
4827 init_jit_module (cfg->domain);
4828 ctx->lmodule = domain_jit_info (cfg->domain)->llvm_module;
4829 method_name = mono_method_full_name (cfg->method, TRUE);
4832 module = ctx->module = ctx->lmodule->module;
4835 LLVM_FAILURE (ctx, "gsharedvt");
4839 static int count = 0;
4842 if (g_getenv ("LLVM_COUNT")) {
4843 if (count == atoi (g_getenv ("LLVM_COUNT"))) {
4844 printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE));
4848 if (count > atoi (g_getenv ("LLVM_COUNT")))
4849 LLVM_FAILURE (ctx, "");
4854 sig = mono_method_signature (cfg->method);
4857 linfo = mono_arch_get_llvm_call_info (cfg, sig);
4859 CHECK_FAILURE (ctx);
4862 linfo->rgctx_arg = TRUE;
4863 method_type = sig_to_llvm_sig_full (ctx, sig, linfo, &sinfo);
4864 CHECK_FAILURE (ctx);
4867 * This maps parameter indexes in the original signature to the indexes in
4868 * the LLVM signature.
4870 ctx->pindexes = sinfo.pindexes;
4872 method = LLVMAddFunction (module, method_name, method_type);
4873 ctx->lmethod = method;
4875 LLVMSetFunctionCallConv (method, LLVMMono1CallConv);
4876 LLVMSetLinkage (method, LLVMPrivateLinkage);
4878 LLVMAddFunctionAttr (method, LLVMUWTable);
4880 if (cfg->compile_aot) {
4881 LLVMSetLinkage (method, LLVMInternalLinkage);
4882 #if LLVM_API_VERSION == 0
4883 /* This causes an assertion in later LLVM versions */
4884 LLVMSetVisibility (method, LLVMHiddenVisibility);
4886 if (ctx->lmodule->external_symbols) {
4887 LLVMSetLinkage (method, LLVMExternalLinkage);
4888 LLVMSetVisibility (method, LLVMHiddenVisibility);
4891 LLVMSetLinkage (method, LLVMPrivateLinkage);
4894 if (cfg->method->save_lmf)
4895 LLVM_FAILURE (ctx, "lmf");
4897 if (sig->pinvoke && cfg->method->wrapper_type != MONO_WRAPPER_RUNTIME_INVOKE)
4898 LLVM_FAILURE (ctx, "pinvoke signature");
4900 header = cfg->header;
4901 for (i = 0; i < header->num_clauses; ++i) {
4902 clause = &header->clauses [i];
4903 if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE)
4904 LLVM_FAILURE (ctx, "non-finally/catch clause.");
4906 if (header->num_clauses || (cfg->method->iflags & METHOD_IMPL_ATTRIBUTE_NOINLINING))
4907 /* We can't handle inlined methods with clauses */
4908 LLVMAddFunctionAttr (method, LLVMNoInlineAttribute);
4910 if (linfo->rgctx_arg) {
4911 ctx->rgctx_arg = LLVMGetParam (method, sinfo.rgctx_arg_pindex);
4913 * We mark the rgctx parameter with the inreg attribute, which is mapped to
4914 * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e.
4915 * CC_X86_64_Mono in X86CallingConv.td.
4917 LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute);
4918 LLVMSetValueName (ctx->rgctx_arg, "rgctx");
4920 if (cfg->vret_addr) {
4921 values [cfg->vret_addr->dreg] = LLVMGetParam (method, sinfo.vret_arg_pindex);
4922 LLVMSetValueName (values [cfg->vret_addr->dreg], "vret");
4925 values [cfg->args [0]->dreg] = LLVMGetParam (method, sinfo.this_arg_pindex);
4926 LLVMSetValueName (values [cfg->args [0]->dreg], "this");
4929 names = g_new (char *, sig->param_count);
4930 mono_method_get_param_names (cfg->method, (const char **) names);
4932 for (i = 0; i < sig->param_count; ++i) {
4935 values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, sinfo.pindexes [i]);
4936 if (names [i] && names [i][0] != '\0')
4937 name = g_strdup_printf ("arg_%s", names [i]);
4939 name = g_strdup_printf ("arg_%d", i);
4940 LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name);
4942 if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeByVal)
4943 LLVMAddAttribute (LLVMGetParam (method, sinfo.pindexes [i]), LLVMByValAttribute);
4947 if (ctx->lmodule->emit_dwarf && cfg->compile_aot && mono_debug_enabled ()) {
4948 ctx->minfo = mono_debug_lookup_method (cfg->method);
4949 ctx->dbg_md = emit_dbg_subprogram (ctx, cfg, method, method_name);
4953 for (bb = cfg->bb_entry; bb; bb = bb->next_bb)
4954 max_block_num = MAX (max_block_num, bb->block_num);
4955 ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1);
4957 /* Add branches between non-consecutive bblocks */
4958 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4959 if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) &&
4960 bb->next_bb != bb->last_ins->inst_false_bb) {
4962 MonoInst *inst = mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst));
4963 inst->opcode = OP_BR;
4964 inst->inst_target_bb = bb->last_ins->inst_false_bb;
4965 mono_bblock_add_inst (bb, inst);
4970 * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR
4971 * was later optimized away, so clear these flags, and add them back for the still
4972 * present OP_LDADDR instructions.
4974 for (i = 0; i < cfg->next_vreg; ++i) {
4977 ins = get_vreg_to_inst (cfg, i);
4978 if (ins && ins != cfg->rgctx_var)
4979 ins->flags &= ~MONO_INST_INDIRECT;
4983 * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags.
4985 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
4987 LLVMBuilderRef builder;
4989 char dname_buf[128];
4991 builder = create_builder (ctx);
4993 for (ins = bb->code; ins; ins = ins->next) {
4994 switch (ins->opcode) {
4999 LLVMTypeRef phi_type = llvm_type_to_stack_type (cfg, type_to_llvm_type (ctx, &ins->klass->byval_arg));
5001 CHECK_FAILURE (ctx);
5003 if (ins->opcode == OP_VPHI) {
5004 /* Treat valuetype PHI nodes as operating on the address itself */
5005 g_assert (ins->klass);
5006 phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0);
5010 * Have to precreate these, as they can be referenced by
5011 * earlier instructions.
5013 sprintf (dname_buf, "t%d", ins->dreg);
5015 values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname);
5017 if (ins->opcode == OP_VPHI)
5018 ctx->addresses [ins->dreg] = values [ins->dreg];
5020 g_ptr_array_add (phi_values, values [ins->dreg]);
5023 * Set the expected type of the incoming arguments since these have
5024 * to have the same type.
5026 for (i = 0; i < ins->inst_phi_args [0]; i++) {
5027 int sreg1 = ins->inst_phi_args [i + 1];
5030 ctx->vreg_types [sreg1] = phi_type;
5035 ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT;
5044 * Create an ordering for bblocks, use the depth first order first, then
5045 * put the exception handling bblocks last.
5047 for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) {
5048 bb = cfg->bblocks [bb_index];
5049 if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) {
5050 g_ptr_array_add (bblock_list, bb);
5051 bblocks [bb->block_num].added = TRUE;
5055 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5056 if (!bblocks [bb->block_num].added)
5057 g_ptr_array_add (bblock_list, bb);
5061 * Second pass: generate code.
5063 for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) {
5064 bb = g_ptr_array_index (bblock_list, bb_index);
5066 if (!(bb == cfg->bb_entry || bb->in_count > 0))
5069 process_bb (ctx, bb);
5070 CHECK_FAILURE (ctx);
5073 /* Add incoming phi values */
5074 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5075 GSList *l, *ins_list;
5077 ins_list = bblocks [bb->block_num].phi_nodes;
5079 for (l = ins_list; l; l = l->next) {
5080 PhiNode *node = l->data;
5081 MonoInst *phi = node->phi;
5082 int sreg1 = node->sreg;
5083 LLVMBasicBlockRef in_bb;
5088 in_bb = get_end_bb (ctx, node->in_bb);
5090 if (ctx->unreachable [node->in_bb->block_num])
5093 if (!values [sreg1])
5094 /* Can happen with values in EH clauses */
5095 LLVM_FAILURE (ctx, "incoming phi sreg1");
5097 if (phi->opcode == OP_VPHI) {
5098 g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg]));
5099 LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1);
5101 if (LLVMTypeOf (values [sreg1]) != LLVMTypeOf (values [phi->dreg]))
5103 LLVM_FAILURE (ctx, "incoming phi arg type mismatch");
5104 g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg]));
5105 LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1);
5110 /* Create the SWITCH statements for ENDFINALLY instructions */
5111 for (bb = cfg->bb_entry; bb; bb = bb->next_bb) {
5112 BBInfo *info = &bblocks [bb->block_num];
5114 for (l = info->endfinally_switch_ins_list; l; l = l->next) {
5115 LLVMValueRef switch_ins = l->data;
5116 GSList *bb_list = info->call_handler_return_bbs;
5118 for (i = 0; i < g_slist_length (bb_list); ++i)
5119 LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data);
5123 if (cfg->verbose_level > 1)
5124 mono_llvm_dump_value (method);
5126 if (cfg->compile_aot)
5127 mark_as_used (ctx->lmodule, method);
5129 if (cfg->compile_aot) {
5130 LLVMValueRef md_args [16];
5131 LLVMValueRef md_node;
5134 method_index = mono_aot_get_method_index (cfg->orig_method);
5135 md_args [0] = LLVMMDString (method_name, strlen (method_name));
5136 md_args [1] = LLVMConstInt (LLVMInt32Type (), method_index, FALSE);
5137 md_node = LLVMMDNode (md_args, 2);
5138 LLVMAddNamedMetadataOperand (module, "mono.function_indexes", md_node);
5139 //LLVMSetMetadata (method, md_kind, LLVMMDNode (&md_arg, 1));
5142 if (cfg->compile_aot) {
5143 /* Don't generate native code, keep the LLVM IR */
5144 if (cfg->compile_aot && cfg->verbose_level)
5145 printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), method_name);
5147 //LLVMVerifyFunction(method, 0);
5149 mono_llvm_optimize_method (ctx->lmodule->mono_ee, method);
5151 if (cfg->verbose_level > 1)
5152 mono_llvm_dump_value (method);
5154 cfg->native_code = LLVMGetPointerToGlobal (ctx->lmodule->ee, method);
5156 /* Set by emit_cb */
5157 g_assert (cfg->code_len);
5159 /* FIXME: Free the LLVM IL for the function */
5162 if (ctx->lmodule->method_to_lmethod)
5163 g_hash_table_insert (ctx->lmodule->method_to_lmethod, cfg->method, method);
5170 /* Need to add unused phi nodes as they can be referenced by other values */
5171 LLVMBasicBlockRef phi_bb = LLVMAppendBasicBlock (method, "PHI_BB");
5172 LLVMBuilderRef builder;
5174 builder = create_builder (ctx);
5175 LLVMPositionBuilderAtEnd (builder, phi_bb);
5177 for (i = 0; i < phi_values->len; ++i) {
5178 LLVMValueRef v = g_ptr_array_index (phi_values, i);
5179 if (LLVMGetInstructionParent (v) == NULL)
5180 LLVMInsertIntoBuilder (builder, v);
5183 LLVMDeleteFunction (method);
5188 g_free (ctx->addresses);
5189 g_free (ctx->vreg_types);
5190 g_free (ctx->vreg_cli_types);
5191 g_free (ctx->pindexes);
5192 g_free (ctx->is_dead);
5193 g_free (ctx->unreachable);
5194 g_ptr_array_free (phi_values, TRUE);
5195 g_free (ctx->bblocks);
5196 g_hash_table_destroy (ctx->region_to_handler);
5197 g_free (method_name);
5198 g_ptr_array_free (bblock_list, TRUE);
5200 for (l = ctx->builders; l; l = l->next) {
5201 LLVMBuilderRef builder = l->data;
5202 LLVMDisposeBuilder (builder);
5207 mono_native_tls_set_value (current_cfg_tls_id, NULL);
5209 mono_loader_unlock ();
5213 * mono_llvm_emit_call:
5215 * Same as mono_arch_emit_call () for LLVM.
5218 mono_llvm_emit_call (MonoCompile *cfg, MonoCallInst *call)
5221 MonoMethodSignature *sig;
5222 int i, n, stack_size;
5227 sig = call->signature;
5228 n = sig->param_count + sig->hasthis;
5230 call->cinfo = mono_arch_get_llvm_call_info (cfg, sig);
5232 if (cfg->disable_llvm)
5235 if (sig->call_convention == MONO_CALL_VARARG) {
5236 cfg->exception_message = g_strdup ("varargs");
5237 cfg->disable_llvm = TRUE;
5240 for (i = 0; i < n; ++i) {
5243 ainfo = call->cinfo->args + i;
5245 in = call->args [i];
5247 /* Simply remember the arguments */
5248 switch (ainfo->storage) {
5250 case LLVMArgInFPReg: {
5251 MonoType *t = (sig->hasthis && i == 0) ? &mono_get_intptr_class ()->byval_arg : sig->params [i - sig->hasthis];
5254 opcode = mono_type_to_regmove (cfg, t);
5255 if (opcode == OP_FMOVE) {
5256 MONO_INST_NEW (cfg, ins, OP_FMOVE);
5257 ins->dreg = mono_alloc_freg (cfg);
5258 } else if (opcode == OP_LMOVE) {
5259 MONO_INST_NEW (cfg, ins, OP_LMOVE);
5260 ins->dreg = mono_alloc_lreg (cfg);
5262 MONO_INST_NEW (cfg, ins, OP_MOVE);
5263 ins->dreg = mono_alloc_ireg (cfg);
5265 ins->sreg1 = in->dreg;
5268 case LLVMArgVtypeByVal:
5269 case LLVMArgVtypeInReg:
5270 case LLVMArgAsIArgs:
5271 case LLVMArgAsFpArgs:
5272 MONO_INST_NEW (cfg, ins, OP_LLVM_OUTARG_VT);
5273 ins->dreg = mono_alloc_ireg (cfg);
5274 ins->sreg1 = in->dreg;
5275 ins->klass = mono_class_from_mono_type (sig->params [i - sig->hasthis]);
5278 call->cinfo = mono_arch_get_llvm_call_info (cfg, sig);
5279 cfg->exception_message = g_strdup ("ainfo->storage");
5280 cfg->disable_llvm = TRUE;
5284 if (!cfg->disable_llvm) {
5285 MONO_ADD_INS (cfg->cbb, ins);
5286 mono_call_inst_add_outarg_reg (cfg, call, ins->dreg, 0, FALSE);
5291 static unsigned char*
5292 alloc_cb (LLVMValueRef function, int size)
5296 cfg = mono_native_tls_get_value (current_cfg_tls_id);
5300 return mono_domain_code_reserve (cfg->domain, size);
5302 return mono_domain_code_reserve (mono_domain_get (), size);
5307 emitted_cb (LLVMValueRef function, void *start, void *end)
5311 cfg = mono_native_tls_get_value (current_cfg_tls_id);
5313 cfg->code_len = (guint8*)end - (guint8*)start;
5317 exception_cb (void *data)
5320 MonoJitExceptionInfo *ei;
5321 guint32 ei_len, i, j, nested_len, nindex;
5322 gpointer *type_info;
5323 int this_reg, this_offset;
5325 cfg = mono_native_tls_get_value (current_cfg_tls_id);
5329 * data points to a DWARF FDE structure, convert it to our unwind format and
5331 * An alternative would be to save it directly, and modify our unwinder to work
5334 cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset);
5335 if (cfg->verbose_level > 1)
5336 mono_print_unwind_info (cfg->encoded_unwind_ops, cfg->encoded_unwind_ops_len);
5338 /* Count nested clauses */
5340 for (i = 0; i < ei_len; ++i) {
5341 for (j = 0; j < ei_len; ++j) {
5342 gint32 cindex1 = *(gint32*)type_info [i];
5343 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
5344 gint32 cindex2 = *(gint32*)type_info [j];
5345 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
5347 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
5353 cfg->llvm_ex_info = mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo));
5354 cfg->llvm_ex_info_len = ei_len + nested_len;
5355 memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo));
5356 /* Fill the rest of the information from the type info */
5357 for (i = 0; i < ei_len; ++i) {
5358 gint32 clause_index = *(gint32*)type_info [i];
5359 MonoExceptionClause *clause = &cfg->header->clauses [clause_index];
5361 cfg->llvm_ex_info [i].flags = clause->flags;
5362 cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class;
5366 * For nested clauses, the LLVM produced exception info associates the try interval with
5367 * the innermost handler, while mono expects it to be associated with all nesting clauses.
5369 /* FIXME: These should be order with the normal clauses */
5371 for (i = 0; i < ei_len; ++i) {
5372 for (j = 0; j < ei_len; ++j) {
5373 gint32 cindex1 = *(gint32*)type_info [i];
5374 MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1];
5375 gint32 cindex2 = *(gint32*)type_info [j];
5376 MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2];
5378 if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) {
5380 * The try interval comes from the nested clause, everything else from the
5383 memcpy (&cfg->llvm_ex_info [nindex], &cfg->llvm_ex_info [j], sizeof (MonoJitExceptionInfo));
5384 cfg->llvm_ex_info [nindex].try_start = cfg->llvm_ex_info [i].try_start;
5385 cfg->llvm_ex_info [nindex].try_end = cfg->llvm_ex_info [i].try_end;
5390 g_assert (nindex == ei_len + nested_len);
5391 cfg->llvm_this_reg = this_reg;
5392 cfg->llvm_this_offset = this_offset;
5394 /* type_info [i] is cfg mempool allocated, no need to free it */
5401 dlsym_cb (const char *name, void **symbol)
5407 if (!strcmp (name, "__bzero")) {
5408 *symbol = (void*)bzero;
5410 current = mono_dl_open (NULL, 0, NULL);
5413 err = mono_dl_symbol (current, name, symbol);
5415 mono_dl_close (current);
5417 #ifdef MONO_ARCH_HAVE_CREATE_LLVM_NATIVE_THUNK
5418 *symbol = (char*)mono_arch_create_llvm_native_thunk (mono_domain_get (), (guint8*)(*symbol));
5424 AddFunc (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef *param_types, int nparams)
5426 LLVMAddFunction (module, name, LLVMFunctionType (ret_type, param_types, nparams, FALSE));
5430 AddFunc2 (LLVMModuleRef module, const char *name, LLVMTypeRef ret_type, LLVMTypeRef param_type1, LLVMTypeRef param_type2)
5432 LLVMTypeRef param_types [4];
5434 param_types [0] = param_type1;
5435 param_types [1] = param_type2;
5437 AddFunc (module, name, ret_type, param_types, 2);
5441 add_intrinsics (LLVMModuleRef module)
5443 /* Emit declarations of instrinsics */
5445 * It would be nicer to emit only the intrinsics actually used, but LLVM's Module
5446 * type doesn't seem to do any locking.
5449 LLVMTypeRef memset_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
5451 memset_param_count = 5;
5452 memset_func_name = "llvm.memset.p0i8.i32";
5454 LLVMAddFunction (module, memset_func_name, LLVMFunctionType (LLVMVoidType (), memset_params, memset_param_count, FALSE));
5458 LLVMTypeRef memcpy_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () };
5460 memcpy_param_count = 5;
5461 memcpy_func_name = "llvm.memcpy.p0i8.p0i8.i32";
5463 LLVMAddFunction (module, memcpy_func_name, LLVMFunctionType (LLVMVoidType (), memcpy_params, memcpy_param_count, FALSE));
5467 LLVMTypeRef params [] = { LLVMDoubleType () };
5469 LLVMAddFunction (module, "llvm.sin.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5470 LLVMAddFunction (module, "llvm.cos.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5471 LLVMAddFunction (module, "llvm.sqrt.f64", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5473 /* This isn't an intrinsic, instead llvm seems to special case it by name */
5474 LLVMAddFunction (module, "fabs", LLVMFunctionType (LLVMDoubleType (), params, 1, FALSE));
5478 LLVMTypeRef ovf_res_i32 [] = { LLVMInt32Type (), LLVMInt1Type () };
5479 LLVMTypeRef ovf_params_i32 [] = { LLVMInt32Type (), LLVMInt32Type () };
5481 LLVMAddFunction (module, "llvm.sadd.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5482 LLVMAddFunction (module, "llvm.uadd.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5483 LLVMAddFunction (module, "llvm.ssub.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5484 LLVMAddFunction (module, "llvm.usub.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5485 LLVMAddFunction (module, "llvm.smul.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5486 LLVMAddFunction (module, "llvm.umul.with.overflow.i32", LLVMFunctionType (LLVMStructType (ovf_res_i32, 2, FALSE), ovf_params_i32, 2, FALSE));
5490 LLVMTypeRef ovf_res_i64 [] = { LLVMInt64Type (), LLVMInt1Type () };
5491 LLVMTypeRef ovf_params_i64 [] = { LLVMInt64Type (), LLVMInt64Type () };
5493 LLVMAddFunction (module, "llvm.sadd.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5494 LLVMAddFunction (module, "llvm.uadd.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5495 LLVMAddFunction (module, "llvm.ssub.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5496 LLVMAddFunction (module, "llvm.usub.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5497 LLVMAddFunction (module, "llvm.smul.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5498 LLVMAddFunction (module, "llvm.umul.with.overflow.i64", LLVMFunctionType (LLVMStructType (ovf_res_i64, 2, FALSE), ovf_params_i64, 2, FALSE));
5503 LLVMTypeRef arg_types [2];
5504 LLVMTypeRef ret_type;
5506 arg_types [0] = LLVMPointerType (LLVMInt8Type (), 0);
5507 arg_types [1] = LLVMPointerType (LLVMInt8Type (), 0);
5508 ret_type = LLVMInt32Type ();
5510 LLVMAddFunction (module, "mono_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
5512 LLVMAddFunction (module, "llvm_resume_unwind_trampoline", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
5515 /* SSE intrinsics */
5516 #if defined(TARGET_X86) || defined(TARGET_AMD64)
5518 LLVMTypeRef ret_type, arg_types [16];
5521 ret_type = type_to_simd_type (MONO_TYPE_I4);
5522 arg_types [0] = ret_type;
5523 arg_types [1] = ret_type;
5524 AddFunc (module, "llvm.x86.sse41.pminud", ret_type, arg_types, 2);
5525 AddFunc (module, "llvm.x86.sse41.pmaxud", ret_type, arg_types, 2);
5527 ret_type = type_to_simd_type (MONO_TYPE_I2);
5528 arg_types [0] = ret_type;
5529 arg_types [1] = ret_type;
5530 AddFunc (module, "llvm.x86.sse41.pminuw", ret_type, arg_types, 2);
5531 AddFunc (module, "llvm.x86.sse2.pmins.w", ret_type, arg_types, 2);
5532 AddFunc (module, "llvm.x86.sse41.pmaxuw", ret_type, arg_types, 2);
5533 AddFunc (module, "llvm.x86.sse2.padds.w", ret_type, arg_types, 2);
5534 AddFunc (module, "llvm.x86.sse2.psubs.w", ret_type, arg_types, 2);
5535 AddFunc (module, "llvm.x86.sse2.paddus.w", ret_type, arg_types, 2);
5536 AddFunc (module, "llvm.x86.sse2.psubus.w", ret_type, arg_types, 2);
5537 AddFunc (module, "llvm.x86.sse2.pavg.w", ret_type, arg_types, 2);
5538 AddFunc (module, "llvm.x86.sse2.pmulh.w", ret_type, arg_types, 2);
5539 AddFunc (module, "llvm.x86.sse2.pmulhu.w", ret_type, arg_types, 2);
5541 ret_type = type_to_simd_type (MONO_TYPE_I1);
5542 arg_types [0] = ret_type;
5543 arg_types [1] = ret_type;
5544 AddFunc (module, "llvm.x86.sse2.pminu.b", ret_type, arg_types, 2);
5545 AddFunc (module, "llvm.x86.sse2.pmaxu.b", ret_type, arg_types, 2);
5546 AddFunc (module, "llvm.x86.sse2.padds.b", ret_type, arg_types, 2);
5547 AddFunc (module, "llvm.x86.sse2.psubs.b", ret_type, arg_types, 2);
5548 AddFunc (module, "llvm.x86.sse2.paddus.b", ret_type, arg_types, 2);
5549 AddFunc (module, "llvm.x86.sse2.psubus.b", ret_type, arg_types, 2);
5550 AddFunc (module, "llvm.x86.sse2.pavg.b", ret_type, arg_types, 2);
5552 ret_type = type_to_simd_type (MONO_TYPE_R8);
5553 arg_types [0] = ret_type;
5554 arg_types [1] = ret_type;
5555 AddFunc (module, "llvm.x86.sse2.min.pd", ret_type, arg_types, 2);
5556 AddFunc (module, "llvm.x86.sse2.max.pd", ret_type, arg_types, 2);
5557 AddFunc (module, "llvm.x86.sse3.hadd.pd", ret_type, arg_types, 2);
5558 AddFunc (module, "llvm.x86.sse3.hsub.pd", ret_type, arg_types, 2);
5559 AddFunc (module, "llvm.x86.sse3.addsub.pd", ret_type, arg_types, 2);
5561 ret_type = type_to_simd_type (MONO_TYPE_R4);
5562 arg_types [0] = ret_type;
5563 arg_types [1] = ret_type;
5564 AddFunc (module, "llvm.x86.sse.min.ps", ret_type, arg_types, 2);
5565 AddFunc (module, "llvm.x86.sse.max.ps", ret_type, arg_types, 2);
5566 AddFunc (module, "llvm.x86.sse3.hadd.ps", ret_type, arg_types, 2);
5567 AddFunc (module, "llvm.x86.sse3.hsub.ps", ret_type, arg_types, 2);
5568 AddFunc (module, "llvm.x86.sse3.addsub.ps", ret_type, arg_types, 2);
5571 ret_type = type_to_simd_type (MONO_TYPE_I1);
5572 arg_types [0] = type_to_simd_type (MONO_TYPE_I2);
5573 arg_types [1] = type_to_simd_type (MONO_TYPE_I2);
5574 AddFunc (module, "llvm.x86.sse2.packsswb.128", ret_type, arg_types, 2);
5575 AddFunc (module, "llvm.x86.sse2.packuswb.128", ret_type, arg_types, 2);
5576 ret_type = type_to_simd_type (MONO_TYPE_I2);
5577 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
5578 arg_types [1] = type_to_simd_type (MONO_TYPE_I4);
5579 AddFunc (module, "llvm.x86.sse2.packssdw.128", ret_type, arg_types, 2);
5580 AddFunc (module, "llvm.x86.sse41.packusdw", ret_type, arg_types, 2);
5583 ret_type = type_to_simd_type (MONO_TYPE_R8);
5584 arg_types [0] = ret_type;
5585 arg_types [1] = ret_type;
5586 arg_types [2] = LLVMInt8Type ();
5587 AddFunc (module, "llvm.x86.sse2.cmp.pd", ret_type, arg_types, 3);
5588 ret_type = type_to_simd_type (MONO_TYPE_R4);
5589 arg_types [0] = ret_type;
5590 arg_types [1] = ret_type;
5591 arg_types [2] = LLVMInt8Type ();
5592 AddFunc (module, "llvm.x86.sse.cmp.ps", ret_type, arg_types, 3);
5594 /* Conversion ops */
5595 ret_type = type_to_simd_type (MONO_TYPE_R8);
5596 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
5597 AddFunc (module, "llvm.x86.sse2.cvtdq2pd", ret_type, arg_types, 1);
5598 ret_type = type_to_simd_type (MONO_TYPE_R4);
5599 arg_types [0] = type_to_simd_type (MONO_TYPE_I4);
5600 AddFunc (module, "llvm.x86.sse2.cvtdq2ps", ret_type, arg_types, 1);
5601 ret_type = type_to_simd_type (MONO_TYPE_I4);
5602 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
5603 AddFunc (module, "llvm.x86.sse2.cvtpd2dq", ret_type, arg_types, 1);
5604 ret_type = type_to_simd_type (MONO_TYPE_I4);
5605 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
5606 AddFunc (module, "llvm.x86.sse2.cvtps2dq", ret_type, arg_types, 1);
5607 ret_type = type_to_simd_type (MONO_TYPE_R4);
5608 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
5609 AddFunc (module, "llvm.x86.sse2.cvtpd2ps", ret_type, arg_types, 1);
5610 ret_type = type_to_simd_type (MONO_TYPE_R8);
5611 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
5612 AddFunc (module, "llvm.x86.sse2.cvtps2pd", ret_type, arg_types, 1);
5614 ret_type = type_to_simd_type (MONO_TYPE_I4);
5615 arg_types [0] = type_to_simd_type (MONO_TYPE_R8);
5616 AddFunc (module, "llvm.x86.sse2.cvttpd2dq", ret_type, arg_types, 1);
5617 ret_type = type_to_simd_type (MONO_TYPE_I4);
5618 arg_types [0] = type_to_simd_type (MONO_TYPE_R4);
5619 AddFunc (module, "llvm.x86.sse2.cvttps2dq", ret_type, arg_types, 1);
5622 ret_type = type_to_simd_type (MONO_TYPE_R8);
5623 arg_types [0] = ret_type;
5624 AddFunc (module, "llvm.x86.sse2.sqrt.pd", ret_type, arg_types, 1);
5625 ret_type = type_to_simd_type (MONO_TYPE_R4);
5626 arg_types [0] = ret_type;
5627 AddFunc (module, "llvm.x86.sse.sqrt.ps", ret_type, arg_types, 1);
5628 ret_type = type_to_simd_type (MONO_TYPE_R4);
5629 arg_types [0] = ret_type;
5630 AddFunc (module, "llvm.x86.sse.rsqrt.ps", ret_type, arg_types, 1);
5631 ret_type = type_to_simd_type (MONO_TYPE_R4);
5632 arg_types [0] = ret_type;
5633 AddFunc (module, "llvm.x86.sse.rcp.ps", ret_type, arg_types, 1);
5636 ret_type = type_to_simd_type (MONO_TYPE_I2);
5637 arg_types [0] = ret_type;
5638 arg_types [1] = LLVMInt32Type ();
5639 AddFunc (module, "llvm.x86.sse2.psrli.w", ret_type, arg_types, 2);
5640 AddFunc (module, "llvm.x86.sse2.psrai.w", ret_type, arg_types, 2);
5641 AddFunc (module, "llvm.x86.sse2.pslli.w", ret_type, arg_types, 2);
5642 ret_type = type_to_simd_type (MONO_TYPE_I4);
5643 arg_types [0] = ret_type;
5644 arg_types [1] = LLVMInt32Type ();
5645 AddFunc (module, "llvm.x86.sse2.psrli.d", ret_type, arg_types, 2);
5646 AddFunc (module, "llvm.x86.sse2.psrai.d", ret_type, arg_types, 2);
5647 AddFunc (module, "llvm.x86.sse2.pslli.d", ret_type, arg_types, 2);
5648 ret_type = type_to_simd_type (MONO_TYPE_I8);
5649 arg_types [0] = ret_type;
5650 arg_types [1] = LLVMInt32Type ();
5651 AddFunc (module, "llvm.x86.sse2.psrli.q", ret_type, arg_types, 2);
5652 AddFunc (module, "llvm.x86.sse2.pslli.q", ret_type, arg_types, 2);
5655 ret_type = LLVMInt32Type ();
5656 arg_types [0] = type_to_simd_type (MONO_TYPE_I1);
5657 AddFunc (module, "llvm.x86.sse2.pmovmskb.128", ret_type, arg_types, 1);
5660 AddFunc (module, "llvm.x86.sse2.pause", LLVMVoidType (), NULL, 0);
5663 /* Load/Store intrinsics */
5665 LLVMTypeRef arg_types [5];
5669 for (i = 1; i <= 8; i *= 2) {
5670 arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0);
5671 arg_types [1] = LLVMInt32Type ();
5672 arg_types [2] = LLVMInt1Type ();
5673 #if LLVM_API_VERSION >= 4
5674 arg_types [3] = LLVMInt32Type ();
5676 sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8);
5677 LLVMAddFunction (module, name, LLVMFunctionType (LLVMIntType (i * 8), arg_types, 3 + EXTRA_MONO_LOAD_STORE_ARGS, FALSE));
5679 arg_types [0] = LLVMIntType (i * 8);
5680 arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0);
5681 arg_types [2] = LLVMInt32Type ();
5682 arg_types [3] = LLVMInt1Type ();
5683 #if LLVM_API_VERSION >= 4
5684 arg_types [4] = LLVMInt32Type ();
5686 sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8);
5687 LLVMAddFunction (module, name, LLVMFunctionType (LLVMVoidType (), arg_types, 4 + EXTRA_MONO_LOAD_STORE_ARGS, FALSE));
5693 add_types (MonoLLVMModule *lmodule)
5695 lmodule->ptr_type = LLVMPointerType (sizeof (gpointer) == 8 ? LLVMInt64Type () : LLVMInt32Type (), 0);
5699 mono_llvm_init (void)
5701 mono_native_tls_alloc (¤t_cfg_tls_id, NULL);
5705 init_jit_module (MonoDomain *domain)
5707 MonoJitICallInfo *info;
5708 MonoJitDomainInfo *dinfo;
5709 MonoLLVMModule *module;
5712 dinfo = domain_jit_info (domain);
5713 if (dinfo->llvm_module)
5716 mono_loader_lock ();
5718 if (dinfo->llvm_module) {
5719 mono_loader_unlock ();
5723 module = g_new0 (MonoLLVMModule, 1);
5725 name = g_strdup_printf ("mono-%s", domain->friendly_name);
5726 module->module = LLVMModuleCreateWithName (name);
5728 module->mono_ee = mono_llvm_create_ee (LLVMCreateModuleProviderForExistingModule (module->module), alloc_cb, emitted_cb, exception_cb, dlsym_cb, &module->ee);
5730 add_intrinsics (module->module);
5733 module->llvm_types = g_hash_table_new (NULL, NULL);
5735 info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline");
5737 LLVMAddGlobalMapping (module->ee, LLVMGetNamedFunction (module->module, "llvm_resume_unwind_trampoline"), (void*)info->func);
5739 mono_memory_barrier ();
5741 dinfo->llvm_module = module;
5743 mono_loader_unlock ();
5747 mono_llvm_cleanup (void)
5749 if (aot_module.module)
5750 LLVMDisposeModule (aot_module.module);
5752 LLVMContextDispose (LLVMGetGlobalContext ());
5756 mono_llvm_free_domain_info (MonoDomain *domain)
5758 MonoJitDomainInfo *info = domain_jit_info (domain);
5759 MonoLLVMModule *module = info->llvm_module;
5765 if (module->llvm_types)
5766 g_hash_table_destroy (module->llvm_types);
5768 mono_llvm_dispose_ee (module->mono_ee);
5770 if (module->bb_names) {
5771 for (i = 0; i < module->bb_names_len; ++i)
5772 g_free (module->bb_names [i]);
5773 g_free (module->bb_names);
5775 //LLVMDisposeModule (module->module);
5779 info->llvm_module = NULL;
5783 mono_llvm_create_aot_module (const char *got_symbol, gboolean external_symbols, gboolean emit_dwarf)
5785 /* Delete previous module */
5786 if (aot_module.plt_entries)
5787 g_hash_table_destroy (aot_module.plt_entries);
5788 if (aot_module.module)
5789 LLVMDisposeModule (aot_module.module);
5791 memset (&aot_module, 0, sizeof (aot_module));
5793 aot_module.module = LLVMModuleCreateWithName ("aot");
5794 aot_module.got_symbol = got_symbol;
5795 aot_module.external_symbols = external_symbols;
5796 aot_module.emit_dwarf = emit_dwarf;
5797 /* The first few entries are reserved */
5798 aot_module.max_got_offset = 16;
5800 add_intrinsics (aot_module.module);
5801 add_types (&aot_module);
5805 * We couldn't compute the type of the LLVM global representing the got because
5806 * its size is only known after all the methods have been emitted. So create
5807 * a dummy variable, and replace all uses it with the real got variable when
5808 * its size is known in mono_llvm_emit_aot_module ().
5811 LLVMTypeRef got_type = LLVMArrayType (aot_module.ptr_type, 0);
5813 aot_module.got_var = LLVMAddGlobal (aot_module.module, got_type, "mono_dummy_got");
5814 LLVMSetInitializer (aot_module.got_var, LLVMConstNull (got_type));
5817 /* Add a dummy personality function */
5819 LLVMBasicBlockRef lbb;
5820 LLVMBuilderRef lbuilder;
5821 LLVMValueRef personality;
5823 personality = LLVMAddFunction (aot_module.module, "mono_aot_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE));
5824 LLVMSetLinkage (personality, LLVMInternalLinkage);
5825 lbb = LLVMAppendBasicBlock (personality, "BB0");
5826 lbuilder = LLVMCreateBuilder ();
5827 LLVMPositionBuilderAtEnd (lbuilder, lbb);
5828 LLVMBuildRetVoid (lbuilder);
5829 mark_as_used (&aot_module, personality);
5832 aot_module.llvm_types = g_hash_table_new (NULL, NULL);
5833 aot_module.plt_entries = g_hash_table_new (g_str_hash, g_str_equal);
5834 aot_module.plt_entries_ji = g_hash_table_new (NULL, NULL);
5835 aot_module.method_to_lmethod = g_hash_table_new (NULL, NULL);
5839 * Emit the aot module into the LLVM bitcode file FILENAME.
5842 mono_llvm_emit_aot_module (const char *filename, const char *cu_name)
5844 LLVMTypeRef got_type;
5845 LLVMValueRef real_got;
5846 MonoLLVMModule *module = &aot_module;
5849 * Create the real got variable and replace all uses of the dummy variable with
5852 got_type = LLVMArrayType (aot_module.ptr_type, module->max_got_offset + 1);
5853 real_got = LLVMAddGlobal (aot_module.module, got_type, aot_module.got_symbol);
5854 LLVMSetInitializer (real_got, LLVMConstNull (got_type));
5855 if (module->external_symbols) {
5856 LLVMSetLinkage (real_got, LLVMExternalLinkage);
5857 LLVMSetVisibility (real_got, LLVMHiddenVisibility);
5859 LLVMSetLinkage (real_got, LLVMInternalLinkage);
5861 mono_llvm_replace_uses_of (aot_module.got_var, real_got);
5863 mark_as_used (&aot_module, real_got);
5865 /* Delete the dummy got so it doesn't become a global */
5866 LLVMDeleteGlobal (aot_module.got_var);
5868 emit_llvm_used (&aot_module);
5869 emit_dbg_info (&aot_module, filename, cu_name);
5871 /* Replace PLT entries for directly callable methods with the methods themselves */
5873 GHashTableIter iter;
5875 LLVMValueRef callee;
5877 g_hash_table_iter_init (&iter, aot_module.plt_entries_ji);
5878 while (g_hash_table_iter_next (&iter, (void**)&ji, (void**)&callee)) {
5879 if (mono_aot_is_direct_callable (ji)) {
5880 LLVMValueRef lmethod;
5882 lmethod = g_hash_table_lookup (module->method_to_lmethod, ji->data.method);
5883 /* The types might not match because the caller might pass an rgctx */
5884 if (lmethod && LLVMTypeOf (callee) == LLVMTypeOf (lmethod)) {
5885 mono_llvm_replace_uses_of (callee, lmethod);
5886 mono_aot_mark_unused_llvm_plt_entry (ji);
5896 if (LLVMVerifyModule (aot_module.module, LLVMReturnStatusAction, &verifier_err)) {
5897 g_assert_not_reached ();
5902 LLVMWriteBitcodeToFile (aot_module.module, filename);
5907 md_string (const char *s)
5909 return LLVMMDString (s, strlen (s));
5912 /* Debugging support */
5915 emit_dbg_info (MonoLLVMModule *lmodule, const char *filename, const char *cu_name)
5917 LLVMModuleRef module = lmodule->module;
5918 LLVMValueRef args [16], cu_args [16], cu, ver;
5920 char *build_info, *s, *dir;
5923 * This can only be enabled when LLVM code is emitted into a separate object
5924 * file, since the AOT compiler also emits dwarf info,
5925 * and the abbrev indexes will not be correct since llvm has added its own
5928 if (!lmodule->emit_dwarf)
5932 * Emit dwarf info in the form of LLVM metadata. There is some
5933 * out-of-date documentation at:
5934 * http://llvm.org/docs/SourceLevelDebugging.html
5935 * but most of this was gathered from the llvm and
5940 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_TAG_compile_unit, FALSE);
5941 /* CU name/compilation dir */
5942 dir = g_path_get_dirname (filename);
5943 args [0] = LLVMMDString (cu_name, strlen (cu_name));
5944 args [1] = LLVMMDString (dir, strlen (dir));
5945 cu_args [n_cuargs ++] = LLVMMDNode (args, 2);
5948 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), DW_LANG_C99, FALSE);
5950 build_info = mono_get_runtime_build_info ();
5951 s = g_strdup_printf ("Mono AOT Compiler %s (LLVM)", build_info);
5952 cu_args [n_cuargs ++] = LLVMMDString (s, strlen (s));
5953 g_free (build_info);
5955 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5957 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
5958 /* Runtime version */
5959 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
5961 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5962 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5964 if (lmodule->subprogram_mds) {
5968 mds = g_new0 (LLVMValueRef, lmodule->subprogram_mds->len);
5969 for (i = 0; i < lmodule->subprogram_mds->len; ++i)
5970 mds [i] = g_ptr_array_index (lmodule->subprogram_mds, i);
5971 cu_args [n_cuargs ++] = LLVMMDNode (mds, lmodule->subprogram_mds->len);
5973 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5976 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5977 /* Imported modules */
5978 cu_args [n_cuargs ++] = LLVMMDNode (args, 0);
5980 cu_args [n_cuargs ++] = LLVMMDString ("", strlen (""));
5981 /* DebugEmissionKind = FullDebug */
5982 cu_args [n_cuargs ++] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5983 cu = LLVMMDNode (cu_args, n_cuargs);
5984 LLVMAddNamedMetadataOperand (module, "llvm.dbg.cu", cu);
5986 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5987 args [1] = LLVMMDString ("Dwarf Version", strlen ("Dwarf Version"));
5988 args [2] = LLVMConstInt (LLVMInt32Type (), 2, FALSE);
5989 ver = LLVMMDNode (args, 3);
5990 LLVMAddNamedMetadataOperand (module, "llvm.module.flags", ver);
5992 args [0] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5993 args [1] = LLVMMDString ("Debug Info Version", strlen ("Debug Info Version"));
5994 args [2] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
5995 ver = LLVMMDNode (args, 3);
5996 LLVMAddNamedMetadataOperand (module, "llvm.module.flags", ver);
6000 emit_dbg_subprogram (EmitContext *ctx, MonoCompile *cfg, LLVMValueRef method, const char *name)
6002 MonoLLVMModule *module = ctx->lmodule;
6003 MonoDebugMethodInfo *minfo = ctx->minfo;
6004 char *source_file, *dir, *filename;
6005 LLVMValueRef md, args [16], ctx_args [16], md_args [64], type_args [16], ctx_md, type_md;
6013 mono_debug_symfile_get_line_numbers_full (minfo, &source_file, NULL, &n_il_offsets, &il_offsets, &line_numbers, NULL, NULL, NULL, NULL);
6015 source_file = g_strdup ("<unknown>");
6016 dir = g_path_get_dirname (source_file);
6017 filename = g_path_get_basename (source_file);
6019 ctx_args [0] = LLVMConstInt (LLVMInt32Type (), 0x29, FALSE);
6020 args [0] = md_string (filename);
6021 args [1] = md_string (dir);
6022 ctx_args [1] = LLVMMDNode (args, 2);
6023 ctx_md = LLVMMDNode (ctx_args, 2);
6025 type_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subroutine_type, FALSE);
6026 type_args [1] = NULL;
6027 type_args [2] = NULL;
6028 type_args [3] = LLVMMDString ("", 0);
6029 type_args [4] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6030 type_args [5] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
6031 type_args [6] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
6032 type_args [7] = LLVMConstInt (LLVMInt64Type (), 0, FALSE);
6033 type_args [8] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6034 type_args [9] = NULL;
6035 type_args [10] = NULL;
6036 type_args [11] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6037 type_args [12] = NULL;
6038 type_args [13] = NULL;
6039 type_args [14] = NULL;
6040 type_md = LLVMMDNode (type_args, 14);
6042 /* http://llvm.org/docs/SourceLevelDebugging.html#subprogram-descriptors */
6043 md_args [0] = LLVMConstInt (LLVMInt32Type (), DW_TAG_subprogram, FALSE);
6044 /* Source directory + file pair */
6045 args [0] = md_string (filename);
6046 args [1] = md_string (dir);
6047 md_args [1] = LLVMMDNode (args ,2);
6048 md_args [2] = ctx_md;
6049 md_args [3] = md_string (cfg->method->name);
6050 md_args [4] = md_string (name);
6051 md_args [5] = md_string (name);
6054 md_args [6] = LLVMConstInt (LLVMInt32Type (), line_numbers [0], FALSE);
6056 md_args [6] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6058 md_args [7] = type_md;
6060 md_args [8] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6062 md_args [9] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6064 md_args [10] = LLVMConstInt (LLVMInt32Type (), 0, FALSE);
6065 /* Index into a virtual function */
6066 md_args [11] = NULL;
6067 md_args [12] = NULL;
6069 md_args [13] = LLVMConstInt (LLVMInt1Type (), 0, FALSE);
6071 md_args [14] = LLVMConstInt (LLVMInt1Type (), 1, FALSE);
6072 /* Pointer to LLVM function */
6073 md_args [15] = method;
6074 /* Function template parameter */
6075 md_args [16] = NULL;
6076 /* Function declaration descriptor */
6077 md_args [17] = NULL;
6078 /* List of function variables */
6079 md_args [18] = LLVMMDNode (args, 0);
6081 md_args [19] = LLVMConstInt (LLVMInt32Type (), 1, FALSE);
6082 md = LLVMMDNode (md_args, 20);
6084 if (!module->subprogram_mds)
6085 module->subprogram_mds = g_ptr_array_new ();
6086 g_ptr_array_add (module->subprogram_mds, md);
6090 g_free (source_file);
6091 g_free (il_offsets);
6092 g_free (line_numbers);
6098 emit_dbg_loc (EmitContext *ctx, LLVMBuilderRef builder, const unsigned char *cil_code)
6100 MonoCompile *cfg = ctx->cfg;
6102 if (ctx->minfo && cil_code && cil_code >= cfg->header->code && cil_code < cfg->header->code + cfg->header->code_size) {
6103 MonoDebugSourceLocation *loc;
6104 LLVMValueRef loc_md, md_args [16];
6107 loc = mono_debug_symfile_lookup_location (ctx->minfo, cil_code - cfg->header->code);
6111 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->row, FALSE);
6112 md_args [nmd_args ++] = LLVMConstInt (LLVMInt32Type (), loc->column, FALSE);
6113 md_args [nmd_args ++] = ctx->dbg_md;
6114 md_args [nmd_args ++] = NULL;
6115 loc_md = LLVMMDNode (md_args, nmd_args);
6116 LLVMSetCurrentDebugLocation (builder, loc_md);
6117 mono_debug_symfile_free_location (loc);
6124 - Emit LLVM IR from the mono IR using the LLVM C API.
6125 - The original arch specific code remains, so we can fall back to it if we run
6126 into something we can't handle.
6130 A partial list of issues:
6131 - Handling of opcodes which can throw exceptions.
6133 In the mono JIT, these are implemented using code like this:
6140 push throw_pos - method
6141 call <exception trampoline>
6143 The problematic part is push throw_pos - method, which cannot be represented
6144 in the LLVM IR, since it does not support label values.
6145 -> this can be implemented in AOT mode using inline asm + labels, but cannot
6146 be implemented in JIT mode ?
6147 -> a possible but slower implementation would use the normal exception
6148 throwing code but it would need to control the placement of the throw code
6149 (it needs to be exactly after the compare+branch).
6150 -> perhaps add a PC offset intrinsics ?
6152 - efficient implementation of .ovf opcodes.
6154 These are currently implemented as:
6155 <ins which sets the condition codes>
6158 Some overflow opcodes are now supported by LLVM SVN.
6160 - exception handling, unwinding.
6161 - SSA is disabled for methods with exception handlers
6162 - How to obtain unwind info for LLVM compiled methods ?
6163 -> this is now solved by converting the unwind info generated by LLVM
6165 - LLVM uses the c++ exception handling framework, while we use our home grown
6166 code, and couldn't use the c++ one:
6167 - its not supported under VC++, other exotic platforms.
6168 - it might be impossible to support filter clauses with it.
6172 The trampolines need a predictable call sequence, since they need to disasm
6173 the calling code to obtain register numbers / offsets.
6175 LLVM currently generates this code in non-JIT mode:
6176 mov -0x98(%rax),%eax
6178 Here, the vtable pointer is lost.
6179 -> solution: use one vtable trampoline per class.
6181 - passing/receiving the IMT pointer/RGCTX.
6182 -> solution: pass them as normal arguments ?
6186 LLVM does not allow the specification of argument registers etc. This means
6187 that all calls are made according to the platform ABI.
6189 - passing/receiving vtypes.
6191 Vtypes passed/received in registers are handled by the front end by using
6192 a signature with scalar arguments, and loading the parts of the vtype into those
6195 Vtypes passed on the stack are handled using the 'byval' attribute.
6199 Supported though alloca, we need to emit the load/store code.
6203 The mono JIT uses pointer sized iregs/double fregs, while LLVM uses precisely
6204 typed registers, so we have to keep track of the precise LLVM type of each vreg.
6205 This is made easier because the IR is already in SSA form.
6206 An additional problem is that our IR is not consistent with types, i.e. i32/ia64
6207 types are frequently used incorrectly.
6212 Emit LLVM bytecode into a .bc file, compile it using llc into a .s file, then
6213 append the AOT data structures to that file. For methods which cannot be
6214 handled by LLVM, the normal JIT compiled versions are used.
6217 /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like:
6218 * - each bblock should end with a branch
6219 * - setting the return value, making cfg->ret non-volatile
6220 * - avoid some transformations in the JIT which make it harder for us to generate
6222 * - use pointer types to help optimizations.