X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Fmini%2Fmini-llvm.c;h=173fe550da2fec44743a064907045a4dda904476;hb=45890e8b46ac438d2b8ccc1bd0d74eea31870de1;hp=2178dd9d5271a055e82a3a5f4c469c46e7c78aa8;hpb=e1aa6e05eb6994e8582140aae15fd90ef73f15be;p=mono.git diff --git a/mono/mini/mini-llvm.c b/mono/mini/mini-llvm.c index 2178dd9d527..173fe550da2 100644 --- a/mono/mini/mini-llvm.c +++ b/mono/mini/mini-llvm.c @@ -27,7 +27,7 @@ */ typedef struct { LLVMModuleRef module; - LLVMValueRef throw, throw_corlib_exception; + LLVMValueRef throw, rethrow, throw_corlib_exception; GHashTable *llvm_types; LLVMValueRef got_var; const char *got_symbol; @@ -46,7 +46,13 @@ typedef struct { * needs to branch to in ENDFINALLY. */ GSList *call_handler_return_bbs; - LLVMValueRef endfinally_switch; + /* + * If this bblock is the start of a finally clause, this is the bblock that + * CALL_HANDLER needs to branch to. + */ + LLVMBasicBlockRef call_handler_target_bb; + /* The list of switch statements generated by ENDFINALLY instructions */ + GSList *endfinally_switch_ins_list; GSList *phi_nodes; } BBInfo; @@ -74,6 +80,11 @@ typedef struct { GHashTable *region_to_handler; LLVMBuilderRef alloca_builder; LLVMValueRef last_alloca; + LLVMValueRef rgctx_arg; + LLVMTypeRef *vreg_types; + gboolean *is_dead; + gboolean *unreachable; + int *pindexes; char temp_name [32]; } EmitContext; @@ -125,6 +136,12 @@ llvm_ins_info[] = { #define TRACE_FAILURE(msg) #endif +#ifdef TARGET_X86 +#define IS_TARGET_X86 1 +#else +#define IS_TARGET_X86 0 +#endif + #define LLVM_FAILURE(ctx, reason) do { \ TRACE_FAILURE (reason); \ (ctx)->cfg->exception_message = g_strdup (reason); \ @@ -167,6 +184,13 @@ static LLVMExecutionEngineRef ee; static guint32 current_cfg_tls_id; static MonoLLVMModule jit_module, aot_module; +static gboolean jit_module_inited; +static int memset_param_count, memcpy_param_count; +static const char *memset_func_name; +static const char *memcpy_func_name; +static const char *eh_selector_name; + +static void init_jit_module (void); /* * IntPtrType: @@ -599,7 +623,9 @@ simd_op_to_intrins (int opcode) case OP_PMINW_UN: return "llvm.x86.sse41.pminuw"; case OP_PMINB_UN: - return "llvm.x86.sse41.pminub"; + return "llvm.x86.sse2.pminu.b"; + case OP_PMINW: + return "llvm.x86.sse2.pmins.w"; case OP_MAXPD: return "llvm.x86.sse2.max.pd"; case OP_MAXPS: @@ -609,7 +635,17 @@ simd_op_to_intrins (int opcode) case OP_PMAXW_UN: return "llvm.x86.sse41.pmaxuw"; case OP_PMAXB_UN: - return "llvm.x86.sse41.pmaxub"; + return "llvm.x86.sse2.pmaxu.b"; + case OP_PCMPEQB: + return "llvm.x86.sse2.pcmpeq.b"; + case OP_PCMPEQW: + return "llvm.x86.sse2.pcmpeq.w"; + case OP_PCMPEQD: + return "llvm.x86.sse2.pcmpeq.d"; + case OP_PCMPEQQ: + return "llvm.x86.sse41.pcmpeqq"; + case OP_PCMPGTB: + return "llvm.x86.sse2.pcmpgt.b"; #endif default: g_assert_not_reached (); @@ -617,6 +653,39 @@ simd_op_to_intrins (int opcode) } } +static LLVMTypeRef +simd_op_to_llvm_type (int opcode) +{ +#if defined(TARGET_X86) || defined(TARGET_AMD64) + switch (opcode) { + case OP_EXTRACT_R8: + case OP_EXPAND_R8: + return LLVMVectorType (LLVMDoubleType (), 2); + case OP_EXTRACT_I8: + case OP_EXPAND_I8: + return LLVMVectorType (LLVMInt64Type (), 2); + case OP_EXTRACT_I4: + case OP_EXPAND_I4: + return LLVMVectorType (LLVMInt32Type (), 4); + case OP_EXTRACT_I2: + case OP_EXTRACT_U2: + case OP_EXPAND_I2: + return LLVMVectorType (LLVMInt16Type (), 8); + case OP_EXTRACT_I1: + case OP_EXTRACT_U1: + case OP_EXPAND_I1: + return LLVMVectorType (LLVMInt8Type (), 16); + case OP_EXPAND_R4: + return LLVMVectorType (LLVMFloatType (), 4); + default: + g_assert_not_reached (); + return NULL; + } +#else + return NULL; +#endif +} + /* * get_bb: * @@ -709,6 +778,8 @@ convert_full (EmitContext *ctx, LLVMValueRef v, LLVMTypeRef dtype, gboolean is_u return LLVMBuildTrunc (ctx->builder, v, dtype, ""); if (stype == LLVMInt32Type () && (dtype == LLVMInt16Type () || dtype == LLVMInt8Type ())) return LLVMBuildTrunc (ctx->builder, v, dtype, ""); + if (stype == LLVMInt16Type () && dtype == LLVMInt8Type ()) + return LLVMBuildTrunc (ctx->builder, v, dtype, ""); if (stype == LLVMDoubleType () && dtype == LLVMFloatType ()) return LLVMBuildFPTrunc (ctx->builder, v, dtype, ""); @@ -758,7 +829,7 @@ emit_volatile_load (EmitContext *ctx, int vreg) * Might have to zero extend since llvm doesn't have * unsigned types. */ - if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2) + if (t->type == MONO_TYPE_U1 || t->type == MONO_TYPE_U2 || t->type == MONO_TYPE_CHAR || t->type == MONO_TYPE_BOOLEAN) v = LLVMBuildZExt (ctx->builder, v, LLVMInt32Type (), ""); else if (t->type == MONO_TYPE_U8) v = LLVMBuildZExt (ctx->builder, v, LLVMInt64Type (), ""); @@ -783,21 +854,36 @@ emit_volatile_store (EmitContext *ctx, int vreg) } } +typedef struct { + /* + * Maps parameter indexes in the original signature to parameter indexes + * in the LLVM signature. + */ + int *pindexes; + /* The indexes of various special arguments in the LLVM signature */ + int vret_arg_pindex, this_arg_pindex, rgctx_arg_pindex, imt_arg_pindex; +} LLVMSigInfo; + /* - * sig_to_llvm_sig: + * sig_to_llvm_sig_full: * * Return the LLVM signature corresponding to the mono signature SIG using the - * calling convention information in CINFO. + * calling convention information in CINFO. Return parameter mapping information in SINFO. */ static LLVMTypeRef -sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo) +sig_to_llvm_sig_full (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo, + LLVMSigInfo *sinfo) { LLVMTypeRef ret_type; LLVMTypeRef *param_types = NULL; LLVMTypeRef res; - int i, j, pindex; + int i, j, pindex, vret_arg_pindex = 0; + int *pindexes; gboolean vretaddr = FALSE; + if (sinfo) + memset (sinfo, 0, sizeof (LLVMSigInfo)); + ret_type = type_to_llvm_type (ctx, sig->ret); CHECK_FAILURE (ctx); @@ -814,17 +900,59 @@ sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo } else if (cinfo && MONO_TYPE_ISSTRUCT (sig->ret)) { g_assert (cinfo->ret.storage == LLVMArgVtypeRetAddr); vretaddr = TRUE; + ret_type = LLVMVoidType (); } - param_types = g_new0 (LLVMTypeRef, (sig->param_count * 2) + 2); + pindexes = g_new0 (int, sig->param_count); + param_types = g_new0 (LLVMTypeRef, (sig->param_count * 2) + 3); pindex = 0; + if (cinfo && cinfo->rgctx_arg) { + if (sinfo) + sinfo->rgctx_arg_pindex = pindex; + param_types [pindex] = IntPtrType (); + pindex ++; + } + if (cinfo && cinfo->imt_arg && IS_LLVM_MONO_BRANCH) { + if (sinfo) + sinfo->imt_arg_pindex = pindex; + param_types [pindex] = IntPtrType (); + pindex ++; + } if (vretaddr) { - ret_type = LLVMVoidType (); + /* Compute the index in the LLVM signature where the vret arg needs to be passed */ + vret_arg_pindex = pindex; + if (cinfo->vret_arg_index == 1) { + /* Add the slots consumed by the first argument */ + LLVMArgInfo *ainfo = &cinfo->args [0]; + switch (ainfo->storage) { + case LLVMArgVtypeInReg: + for (j = 0; j < 2; ++j) { + if (ainfo->pair_storage [j] == LLVMArgInIReg) + vret_arg_pindex ++; + } + break; + default: + vret_arg_pindex ++; + } + } + + if (sinfo) + sinfo->vret_arg_pindex = vret_arg_pindex; + } + + if (vretaddr && vret_arg_pindex == pindex) + param_types [pindex ++] = IntPtrType (); + if (sig->hasthis) { + if (sinfo) + sinfo->this_arg_pindex = pindex; param_types [pindex ++] = IntPtrType (); } - if (sig->hasthis) + if (vretaddr && vret_arg_pindex == pindex) param_types [pindex ++] = IntPtrType (); for (i = 0; i < sig->param_count; ++i) { + if (vretaddr && vret_arg_pindex == pindex) + param_types [pindex ++] = IntPtrType (); + pindexes [i] = pindex; if (cinfo && cinfo->args [i + sig->hasthis].storage == LLVMArgVtypeInReg) { for (j = 0; j < 2; ++j) { switch (cinfo->args [i + sig->hasthis].pair_storage [j]) { @@ -846,11 +974,20 @@ sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo param_types [pindex ++] = type_to_llvm_arg_type (ctx, sig->params [i]); } } + if (vretaddr && vret_arg_pindex == pindex) + param_types [pindex ++] = IntPtrType (); + CHECK_FAILURE (ctx); res = LLVMFunctionType (ret_type, param_types, pindex, FALSE); g_free (param_types); + if (sinfo) { + sinfo->pindexes = pindexes; + } else { + g_free (pindexes); + } + return res; FAILURE: @@ -859,6 +996,12 @@ sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig, LLVMCallInfo *cinfo return NULL; } +static LLVMTypeRef +sig_to_llvm_sig (EmitContext *ctx, MonoMethodSignature *sig) +{ + return sig_to_llvm_sig_full (ctx, sig, NULL, NULL); +} + /* * LLVMFunctionType1: * @@ -940,20 +1083,59 @@ get_plt_entry (EmitContext *ctx, LLVMTypeRef llvm_sig, MonoJumpInfoType type, gc if (!callee_name) return NULL; + if (ctx->cfg->compile_aot) + /* Add a patch so referenced wrappers can be compiled in full aot mode */ + mono_add_patch_info (ctx->cfg, 0, type, data); + // FIXME: Locking callee = g_hash_table_lookup (ctx->lmodule->plt_entries, callee_name); if (!callee) { callee = LLVMAddFunction (ctx->module, callee_name, llvm_sig); + LLVMSetVisibility (callee, LLVMHiddenVisibility); + g_hash_table_insert (ctx->lmodule->plt_entries, (char*)callee_name, callee); } return callee; } +static int +get_handler_clause (MonoCompile *cfg, MonoBasicBlock *bb) +{ + MonoMethodHeader *header = cfg->header; + MonoExceptionClause *clause; + int i; + + /* Directly */ + if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY)) + return (bb->region >> 8) - 1; + + /* Indirectly */ + for (i = 0; i < header->num_clauses; ++i) { + clause = &header->clauses [i]; + + if (MONO_OFFSET_IN_CLAUSE (clause, bb->real_offset) && clause->flags == MONO_EXCEPTION_CLAUSE_NONE) + return i; + } + + return -1; +} + static void -emit_cond_throw_pos (EmitContext *ctx) +set_metadata_flag (LLVMValueRef v, const char *flag_name) { +#if LLVM_CHECK_VERSION (2, 8) + LLVMValueRef md_arg; + int md_kind; + + if (!IS_LLVM_MONO_BRANCH) + return; + + md_kind = LLVMGetMDKindID (flag_name, strlen (flag_name)); + md_arg = LLVMMDString ("mono", 4); + LLVMSetMetadata (v, md_kind, LLVMMDNode (&md_arg, 1)); +#endif } /* @@ -968,12 +1150,12 @@ emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LL MonoCompile *cfg = ctx->cfg; LLVMValueRef lcall; LLVMBuilderRef builder = *builder_ref; + int clause_index; - // FIXME: Nested clauses - if (bb->region != -1 && MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY)) { - MonoMethodHeader *header = mono_method_get_header (cfg->method); - // FIXME: Add a macro for this - int clause_index = (bb->region >> 8) - 1; + clause_index = get_handler_clause (cfg, bb); + + if (clause_index != -1) { + MonoMethodHeader *header = cfg->header; MonoExceptionClause *ec = &header->clauses [clause_index]; MonoBasicBlock *tblock; LLVMBasicBlockRef ex_bb, noex_bb; @@ -1011,10 +1193,116 @@ emit_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, LL return lcall; } +static LLVMValueRef +emit_load (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef addr, const char *name, gboolean is_faulting) +{ + const char *intrins_name; + LLVMValueRef args [16], res; + LLVMTypeRef addr_type; + + if (is_faulting && bb->region != -1 && IS_LLVM_MONO_BRANCH) { + /* + * We handle loads which can fault by calling a mono specific intrinsic + * using an invoke, so they are handled properly inside try blocks. + * We can't use this outside clauses, since LLVM optimizes intrinsics which + * are marked with IntrReadArgMem. + */ + switch (size) { + case 1: + intrins_name = "llvm.mono.load.i8.p0i8"; + break; + case 2: + intrins_name = "llvm.mono.load.i16.p0i16"; + break; + case 4: + intrins_name = "llvm.mono.load.i32.p0i32"; + break; + case 8: + intrins_name = "llvm.mono.load.i64.p0i64"; + break; + default: + g_assert_not_reached (); + } + + addr_type = LLVMTypeOf (addr); + if (addr_type == LLVMPointerType (LLVMDoubleType (), 0) || addr_type == LLVMPointerType (LLVMFloatType (), 0)) + addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), ""); + + args [0] = addr; + args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + args [2] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE); + res = emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->module, intrins_name), args, 3); + + if (addr_type == LLVMPointerType (LLVMDoubleType (), 0)) + res = LLVMBuildBitCast (*builder_ref, res, LLVMDoubleType (), ""); + else if (addr_type == LLVMPointerType (LLVMFloatType (), 0)) + res = LLVMBuildBitCast (*builder_ref, res, LLVMFloatType (), ""); + + return res; + } else { + LLVMValueRef res; + + /* + * We emit volatile loads for loads which can fault, because otherwise + * LLVM will generate invalid code when encountering a load from a + * NULL address. + */ + res = mono_llvm_build_load (*builder_ref, addr, name, is_faulting); + + /* Mark it with a custom metadata */ + /* + if (is_faulting) + set_metadata_flag (res, "mono.faulting.load"); + */ + + return res; + } +} + +static void +emit_store (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, int size, LLVMValueRef value, LLVMValueRef addr, gboolean is_faulting) +{ + const char *intrins_name; + LLVMValueRef args [16]; + + if (is_faulting && bb->region != -1 && IS_LLVM_MONO_BRANCH) { + switch (size) { + case 1: + intrins_name = "llvm.mono.store.i8.p0i8"; + break; + case 2: + intrins_name = "llvm.mono.store.i16.p0i16"; + break; + case 4: + intrins_name = "llvm.mono.store.i32.p0i32"; + break; + case 8: + intrins_name = "llvm.mono.store.i64.p0i64"; + break; + default: + g_assert_not_reached (); + } + + if (LLVMTypeOf (value) == LLVMDoubleType () || LLVMTypeOf (value) == LLVMFloatType ()) { + value = LLVMBuildBitCast (*builder_ref, value, LLVMIntType (size * 8), ""); + addr = LLVMBuildBitCast (*builder_ref, addr, LLVMPointerType (LLVMIntType (size * 8), 0), ""); + } + + args [0] = value; + args [1] = addr; + args [2] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + args [3] = LLVMConstInt (LLVMInt1Type (), TRUE, FALSE); + emit_call (ctx, bb, builder_ref, LLVMGetNamedFunction (ctx->module, intrins_name), args, 4); + } else { + LLVMBuildStore (*builder_ref, value, addr); + } +} + /* * emit_cond_system_exception: * * Emit code to throw the exception EXC_TYPE if the condition CMP is false. + * Might set the ctx exception. */ static void emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *exc_type, LLVMValueRef cmp) @@ -1023,13 +1311,13 @@ emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *ex LLVMBuilderRef builder; MonoClass *exc_class; LLVMValueRef args [2]; - + ex_bb = gen_bb (ctx, "EX_BB"); noex_bb = gen_bb (ctx, "NOEX_BB"); LLVMBuildCondBr (ctx->builder, cmp, ex_bb, noex_bb); - exc_class = mono_class_from_name (mono_defaults.corlib, "System", exc_type); + exc_class = mono_class_from_name (mono_get_corlib (), "System", exc_type); g_assert (exc_class); /* Emit exception throwing code */ @@ -1039,45 +1327,61 @@ emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *ex if (!ctx->lmodule->throw_corlib_exception) { LLVMValueRef callee; LLVMTypeRef sig; - - MonoMethodSignature *throw_sig = mono_metadata_signature_alloc (mono_defaults.corlib, 2); - throw_sig->ret = &mono_defaults.void_class->byval_arg; - throw_sig->params [0] = &mono_defaults.int32_class->byval_arg; - throw_sig->params [1] = &mono_defaults.int32_class->byval_arg; - sig = sig_to_llvm_sig (ctx, throw_sig, NULL); + const char *icall_name; + + MonoMethodSignature *throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 2); + throw_sig->ret = &mono_get_void_class ()->byval_arg; + throw_sig->params [0] = &mono_get_int32_class ()->byval_arg; + if (IS_LLVM_MONO_BRANCH) { + icall_name = "llvm_throw_corlib_exception_abs_trampoline"; + throw_sig->params [1] = &mono_get_intptr_class ()->byval_arg; + } else { + icall_name = "llvm_throw_corlib_exception_trampoline"; + throw_sig->params [1] = &mono_get_int32_class ()->byval_arg; + } + sig = sig_to_llvm_sig (ctx, throw_sig); if (ctx->cfg->compile_aot) { - callee = get_plt_entry (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_corlib_exception"); + callee = get_plt_entry (ctx, sig, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name); } else { - callee = LLVMAddFunction (ctx->module, "throw_corlib_exception", sig_to_llvm_sig (ctx, throw_sig, NULL)); + callee = LLVMAddFunction (ctx->module, "llvm_throw_corlib_exception_trampoline", sig_to_llvm_sig (ctx, throw_sig)); -#ifdef TARGET_X86 - /* - * LLVM generated code doesn't push the arguments, so we need another - * throw trampoline. + /* + * Differences between the LLVM/non-LLVM throw corlib exception trampoline: + * - On x86, LLVM generated code doesn't push the arguments + * - When using the LLVM mono branch, the trampoline takes the throw address as an + * arguments, not a pc offset. */ - LLVMAddGlobalMapping (ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_llvm_throw_corlib_exception")); -#else - LLVMAddGlobalMapping (ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_corlib_exception")); -#endif + LLVMAddGlobalMapping (ee, callee, resolve_patch (ctx->cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name)); } mono_memory_barrier (); ctx->lmodule->throw_corlib_exception = callee; } -#ifdef TARGET_X86 - args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE); -#else - args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE); -#endif - /* - * FIXME: The offset is 0, this is not a problem for exception handling - * in general, because we don't llvm compile methods with handlers, its only - * a problem for line numbers in stack traces. - */ - args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - emit_call (ctx, bb, &builder, ctx->lmodule->throw_corlib_exception, args, 2); + if (IS_TARGET_X86) + args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token - MONO_TOKEN_TYPE_DEF, FALSE); + else + args [0] = LLVMConstInt (LLVMInt32Type (), exc_class->type_token, FALSE); + + if (IS_LLVM_MONO_BRANCH) { + /* + * The LLVM mono branch contains changes so a block address can be passed as an + * argument to a call. + */ + args [1] = LLVMBuildPtrToInt (builder, LLVMBlockAddress (ctx->lmethod, ex_bb), IntPtrType (), ""); + emit_call (ctx, bb, &builder, ctx->lmodule->throw_corlib_exception, args, 2); + } else { + /* + * FIXME: The offset is 0, this is only a problem if the code is inside a clause, + * otherwise only the line numbers in stack traces are incorrect. + */ + if (bb->region != -1 && !IS_LLVM_MONO_BRANCH) + LLVM_FAILURE (ctx, "system-ex-in-region"); + + args [1] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + emit_call (ctx, bb, &builder, ctx->lmodule->throw_corlib_exception, args, 2); + } LLVMBuildUnreachable (builder); @@ -1087,6 +1391,10 @@ emit_cond_system_exception (EmitContext *ctx, MonoBasicBlock *bb, const char *ex ctx->bblocks [bb->block_num].end_bblock = noex_bb; ctx->ex_index ++; + return; + + FAILURE: + return; } /* @@ -1232,7 +1540,7 @@ mark_as_used (LLVMModuleRef module, LLVMValueRef global) * Emit code to load/convert arguments. */ static void -emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder, int *pindexes) +emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder) { int i, pindex; MonoCompile *cfg = ctx->cfg; @@ -1271,7 +1579,7 @@ emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder, int *pindexes) * Emit code to save the argument from the registers to * the real argument. */ - pindex = pindexes [i]; + pindex = ctx->pindexes [i]; regs [0] = LLVMGetParam (ctx->lmethod, pindex); if (ainfo->pair_storage [1] != LLVMArgNone) regs [1] = LLVMGetParam (ctx->lmethod, pindex + 1); @@ -1281,8 +1589,18 @@ emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder, int *pindexes) ctx->addresses [reg] = build_alloca (ctx, sig->params [i]); emit_reg_to_vtype (ctx, builder, sig->params [i], ctx->addresses [reg], ainfo, regs); + + if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) { + /* Treat these as normal values */ + ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], ""); + } } else if (ainfo->storage == LLVMArgVtypeByVal) { - ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, pindexes [i]); + ctx->addresses [reg] = LLVMGetParam (ctx->lmethod, ctx->pindexes [i]); + + if (MONO_CLASS_IS_SIMD (ctx->cfg, mono_class_from_mono_type (sig->params [i]))) { + /* Treat these as normal values */ + ctx->values [reg] = LLVMBuildLoad (builder, ctx->addresses [reg], ""); + } } else { ctx->values [reg] = convert (ctx, ctx->values [reg], llvm_type_to_stack_type (type_to_llvm_type (ctx, sig->params [i]))); } @@ -1296,6 +1614,36 @@ emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder, int *pindexes) if (!MONO_TYPE_ISSTRUCT (sig->params [i])) emit_volatile_store (ctx, cfg->args [i + sig->hasthis]->dreg); + if (sig->hasthis && !cfg->rgctx_var && cfg->generic_sharing_context) { + LLVMValueRef this_alloc; + + /* + * The exception handling code needs the location where the this argument was + * stored for gshared methods. We create a separate alloca to hold it, and mark it + * with the "mono.this" custom metadata to tell llvm that it needs to save its + * location into the LSDA. + */ + this_alloc = mono_llvm_build_alloca (builder, IntPtrType (), LLVMConstInt (LLVMInt32Type (), 1, FALSE), 0, ""); + /* This volatile store will keep the alloca alive */ + mono_llvm_build_store (builder, ctx->values [cfg->args [0]->dreg], this_alloc, TRUE); + + set_metadata_flag (this_alloc, "mono.this"); + } + + if (cfg->rgctx_var) { + LLVMValueRef rgctx_alloc, store; + + /* + * We handle the rgctx arg similarly to the this pointer. + */ + g_assert (ctx->addresses [cfg->rgctx_var->dreg]); + rgctx_alloc = ctx->addresses [cfg->rgctx_var->dreg]; + /* This volatile store will keep the alloca alive */ + store = mono_llvm_build_store (builder, ctx->rgctx_arg, rgctx_alloc, TRUE); + + set_metadata_flag (rgctx_alloc, "mono.this"); + } + /* * For finally clauses, create an indicator variable telling OP_ENDFINALLY whenever * it needs to continue normally, or return back to the exception handling system. @@ -1304,10 +1652,22 @@ emit_entry_bb (EmitContext *ctx, LLVMBuilderRef builder, int *pindexes) if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER)) g_hash_table_insert (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region)), bb); if (bb->region != -1 && (bb->flags & BB_EXCEPTION_HANDLER) && bb->in_scount == 0) { - LLVMValueRef val = LLVMBuildAlloca (builder, LLVMInt32Type (), ""); + char name [128]; + LLVMValueRef val; + + sprintf (name, "finally_ind_bb%d", bb->block_num); + val = LLVMBuildAlloca (builder, LLVMInt32Type (), name); LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), val); ctx->bblocks [bb->block_num].finally_ind = val; + + /* + * Create a new bblock which CALL_HANDLER can branch to, because branching to the + * LLVM bblock containing the call to llvm.eh.selector causes problems for the + * LLVM optimizer passes. + */ + sprintf (name, "BB_%d_CALL_HANDLER_TARGET", bb->block_num); + ctx->bblocks [bb->block_num].call_handler_target_bb = LLVMAppendBasicBlock (ctx->lmethod, name); } } @@ -1326,1678 +1686,1522 @@ mono_personality (void) g_assert_not_reached (); } -/* - * mono_llvm_emit_method: - * - * Emit LLVM IL from the mono IL, and compile it to native code using LLVM. - */ -void -mono_llvm_emit_method (MonoCompile *cfg) +static void +process_call (EmitContext *ctx, MonoBasicBlock *bb, LLVMBuilderRef *builder_ref, MonoInst *ins) { - EmitContext *ctx; - MonoMethodSignature *sig; - MonoBasicBlock *bb; - LLVMTypeRef method_type; - LLVMValueRef method = NULL, debug_alias = NULL; - char *method_name, *debug_name = NULL; - LLVMValueRef *values, *addresses; - LLVMTypeRef *vreg_types; - MonoType **vreg_cli_types; - int i, max_block_num, pindex, bb_index; - int *pindexes = NULL; - gboolean last = FALSE; - GPtrArray *phi_values; - LLVMCallInfo *linfo; + MonoCompile *cfg = ctx->cfg; + LLVMModuleRef module = ctx->module; + LLVMValueRef *values = ctx->values; + LLVMValueRef *addresses = ctx->addresses; + MonoCallInst *call = (MonoCallInst*)ins; + MonoMethodSignature *sig = call->signature; + LLVMValueRef callee, lcall; + LLVMValueRef *args; + LLVMCallInfo *cinfo; GSList *l; - LLVMModuleRef module; - gboolean *is_dead; - gboolean *unreachable; - BBInfo *bblocks; - GPtrArray *bblock_list; - MonoMethodHeader *header; - MonoExceptionClause *clause; + int i, len; + gboolean vretaddr; + LLVMTypeRef llvm_sig; + gpointer target; + gboolean virtual, calli; + LLVMBuilderRef builder = *builder_ref; + LLVMSigInfo sinfo; - /* The code below might acquire the loader lock, so use it for global locking */ - mono_loader_lock (); + if (call->signature->call_convention != MONO_CALL_DEFAULT) + LLVM_FAILURE (ctx, "non-default callconv"); - /* Used to communicate with the callbacks */ - TlsSetValue (current_cfg_tls_id, cfg); + if (call->rgctx_arg_reg && !IS_LLVM_MONO_BRANCH) + LLVM_FAILURE (ctx, "rgctx reg in call"); - ctx = g_new0 (EmitContext, 1); - ctx->cfg = cfg; - ctx->mempool = cfg->mempool; + if (call->rgctx_reg && !IS_LLVM_MONO_BRANCH) { + /* + * It might be possible to support this by creating a static rgctx trampoline, but + * common_call_trampoline () would patch callsites to call the trampoline, which + * would be incorrect if the rgctx arg is computed dynamically. + */ + LLVM_FAILURE (ctx, "rgctx reg"); + } - /* - * This maps vregs to the LLVM instruction defining them - */ - values = g_new0 (LLVMValueRef, cfg->next_vreg); - /* - * This maps vregs for volatile variables to the LLVM instruction defining their - * address. - */ - addresses = g_new0 (LLVMValueRef, cfg->next_vreg); - vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg); - vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg); - phi_values = g_ptr_array_new (); - /* - * This signals whenever the vreg was defined by a phi node with no input vars - * (i.e. all its input bblocks end with NOT_REACHABLE). - */ - is_dead = g_new0 (gboolean, cfg->next_vreg); - /* Whenever the bblock is unreachable */ - unreachable = g_new0 (gboolean, cfg->max_block_num); + cinfo = call->cinfo; + if (call->rgctx_arg_reg) + cinfo->rgctx_arg = TRUE; + if (call->imt_arg_reg) + cinfo->imt_arg = TRUE; - bblock_list = g_ptr_array_new (); + vretaddr = cinfo && cinfo->ret.storage == LLVMArgVtypeRetAddr; - ctx->values = values; - ctx->addresses = addresses; - ctx->vreg_cli_types = vreg_cli_types; - ctx->region_to_handler = g_hash_table_new (NULL, NULL); - - if (cfg->compile_aot) { - ctx->lmodule = &aot_module; - method_name = mono_aot_get_method_name (cfg); - debug_name = mono_aot_get_method_debug_name (cfg); - } else { - ctx->lmodule = &jit_module; - method_name = mono_method_full_name (cfg->method, TRUE); - debug_name = NULL; - } - - module = ctx->module = ctx->lmodule->module; + llvm_sig = sig_to_llvm_sig_full (ctx, sig, cinfo, &sinfo); + CHECK_FAILURE (ctx); -#if 1 - { - static int count = 0; - count ++; + virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE); + calli = (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG); - if (getenv ("LLVM_COUNT")) { - if (count == atoi (getenv ("LLVM_COUNT"))) { - printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE)); - last = TRUE; + /* FIXME: Avoid creating duplicate methods */ + + if (ins->flags & MONO_INST_HAS_METHOD) { + if (virtual) { + callee = NULL; + } else { + if (cfg->compile_aot) { + callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method); + if (!callee) + LLVM_FAILURE (ctx, "can't encode patch"); + } else { + callee = LLVMAddFunction (module, "", llvm_sig); + + target = + mono_create_jit_trampoline_in_domain (mono_domain_get (), + call->method); + LLVMAddGlobalMapping (ee, callee, target); } - if (count > atoi (getenv ("LLVM_COUNT"))) - LLVM_FAILURE (ctx, ""); } - } -#endif + } else if (calli) { + } else { + MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr); - sig = mono_method_signature (cfg->method); - ctx->sig = sig; + if (info) { + /* + MonoJumpInfo ji; - linfo = mono_arch_get_llvm_call_info (cfg, sig); - ctx->linfo = linfo; - CHECK_FAILURE (ctx); + memset (&ji, 0, sizeof (ji)); + ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR; + ji.data.target = info->name; - method_type = sig_to_llvm_sig (ctx, sig, linfo); - CHECK_FAILURE (ctx); - - method = LLVMAddFunction (module, method_name, method_type); - ctx->lmethod = method; + target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE); + */ + if (cfg->compile_aot) { + callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name); + if (!callee) + LLVM_FAILURE (ctx, "can't encode patch"); + } else { + callee = LLVMAddFunction (module, "", llvm_sig); + target = (gpointer)mono_icall_get_wrapper (info); + LLVMAddGlobalMapping (ee, callee, target); + } + } else { + if (cfg->compile_aot) { + callee = NULL; + if (cfg->abs_patches) { + MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr); + if (abs_ji) { + callee = get_plt_entry (ctx, llvm_sig, abs_ji->type, abs_ji->data.target); + if (!callee) + LLVM_FAILURE (ctx, "can't encode patch"); + } + } + if (!callee) + LLVM_FAILURE (ctx, "aot"); + } else { + callee = LLVMAddFunction (module, "", llvm_sig); + target = NULL; + if (cfg->abs_patches) { + MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr); + if (abs_ji) { + /* + * FIXME: Some trampolines might have + * their own calling convention on some platforms. + */ +#ifndef TARGET_AMD64 + if (abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER || abs_ji->type == MONO_PATCH_INFO_MONITOR_EXIT || abs_ji->type == MONO_PATCH_INFO_GENERIC_CLASS_INIT) + LLVM_FAILURE (ctx, "trampoline with own cconv"); +#endif + target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE); + LLVMAddGlobalMapping (ee, callee, target); + } + } + if (!target) + LLVMAddGlobalMapping (ee, callee, (gpointer)call->fptr); + } + } + } - LLVMSetLinkage (method, LLVMPrivateLinkage); + if (virtual) { + int size = sizeof (gpointer); + LLVMValueRef index; - if (cfg->method->save_lmf) - LLVM_FAILURE (ctx, "lmf"); + g_assert (ins->inst_offset % size == 0); + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); - if (sig->pinvoke) - LLVM_FAILURE (ctx, "pinvoke signature"); + /* + * When using the llvm mono branch, we can support IMT directly, otherwise + * we need to call a trampoline. + */ + if (call->method && call->method->klass->flags & TYPE_ATTRIBUTE_INTERFACE && !IS_LLVM_MONO_BRANCH) { +#ifdef MONO_ARCH_HAVE_LLVM_IMT_TRAMPOLINE + if (cfg->compile_aot) { + MonoJumpInfoImtTramp *imt_tramp = g_new0 (MonoJumpInfoImtTramp, 1); + imt_tramp->method = call->method; + imt_tramp->vt_offset = call->inst.inst_offset; - header = mono_method_get_header (cfg->method); - for (i = 0; i < header->num_clauses; ++i) { - clause = &header->clauses [i]; - if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) - LLVM_FAILURE (ctx, "non-finally/catch clause."); + callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_LLVM_IMT_TRAMPOLINE, imt_tramp); + } else { + callee = LLVMAddFunction (module, "", llvm_sig); + target = mono_create_llvm_imt_trampoline (cfg->domain, call->method, call->inst.inst_offset); + LLVMAddGlobalMapping (ee, callee, target); + } +#else + /* No support for passing the IMT argument */ + LLVM_FAILURE (ctx, "imt"); +#endif + } else { + callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0)); + } + } else if (calli) { + callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0)); + } else { + if (ins->flags & MONO_INST_HAS_METHOD) { + } } /* - * This maps parameter indexes in the original signature to the indexes in - * the LLVM signature. + * Collect and convert arguments */ - pindexes = g_new0 (int, sig->param_count); - pindex = 0; - if (cfg->vret_addr) { - values [cfg->vret_addr->dreg] = LLVMGetParam (method, pindex); - pindex ++; + len = sizeof (LLVMValueRef) * ((sig->param_count * 2) + sig->hasthis + vretaddr + call->rgctx_reg); + args = alloca (len); + memset (args, 0, len); + l = call->out_ireg_args; + + if (IS_LLVM_MONO_BRANCH) { + if (call->rgctx_arg_reg) { + g_assert (values [call->rgctx_arg_reg]); + args [sinfo.rgctx_arg_pindex] = values [call->rgctx_arg_reg]; + } + if (call->imt_arg_reg) { + g_assert (values [call->imt_arg_reg]); + args [sinfo.imt_arg_pindex] = values [call->imt_arg_reg]; + } } - if (sig->hasthis) { - values [cfg->args [0]->dreg] = LLVMGetParam (method, pindex); - pindex ++; + + if (vretaddr) { + if (!addresses [call->inst.dreg]) + addresses [call->inst.dreg] = build_alloca (ctx, sig->ret); + args [sinfo.vret_arg_pindex] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), ""); } - for (i = 0; i < sig->param_count; ++i) { - values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, pindex); - pindexes [i] = pindex; - if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeInReg) { - if (linfo->args [i + sig->hasthis].pair_storage [0] != LLVMArgNone) - pindex ++; - if (linfo->args [i + sig->hasthis].pair_storage [1] != LLVMArgNone) - pindex ++; - } else if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeByVal) { - LLVMAddAttribute (LLVMGetParam (method, pindex), LLVMByValAttribute); - pindex ++; + + for (i = 0; i < sig->param_count + sig->hasthis; ++i) { + guint32 regpair; + int reg, pindex; + LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL; + + if (sig->hasthis) { + if (i == 0) + pindex = sinfo.this_arg_pindex; + else + pindex = sinfo.pindexes [i - 1]; } else { - pindex ++; + pindex = sinfo.pindexes [i]; } - } - max_block_num = 0; - for (bb = cfg->bb_entry; bb; bb = bb->next_bb) - max_block_num = MAX (max_block_num, bb->block_num); - ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1); + regpair = (guint32)(gssize)(l->data); + reg = regpair & 0xffffff; + args [pindex] = values [reg]; + if (ainfo->storage == LLVMArgVtypeInReg) { + int j; + LLVMValueRef regs [2]; + guint32 nregs; - /* Add branches between non-consecutive bblocks */ - for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { - if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) && - bb->next_bb != bb->last_ins->inst_false_bb) { - - MonoInst *inst = mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst)); - inst->opcode = OP_BR; - inst->inst_target_bb = bb->last_ins->inst_false_bb; - mono_bblock_add_inst (bb, inst); + g_assert (ainfo); + + g_assert (addresses [reg]); + + emit_vtype_to_reg (ctx, builder, sig->params [i - sig->hasthis], addresses [reg], ainfo, regs, &nregs); + for (j = 0; j < nregs; ++j) + args [pindex ++] = regs [j]; + + // FIXME: alignment + // FIXME: Get rid of the VMOVE + } else if (ainfo->storage == LLVMArgVtypeByVal) { + g_assert (addresses [reg]); + args [pindex] = addresses [reg]; + } else { + g_assert (args [pindex]); + if (i == 0 && sig->hasthis) + args [pindex] = convert (ctx, args [pindex], IntPtrType ()); + else + args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, sig->params [i - sig->hasthis])); } + + l = l->next; } + // FIXME: Align call sites + /* - * Make a first pass over the code to precreate PHI nodes. + * Emit the call */ - for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { - MonoInst *ins; - LLVMBuilderRef builder; - char *dname; - char dname_buf[128]; - - builder = create_builder (ctx); - for (ins = bb->code; ins; ins = ins->next) { - switch (ins->opcode) { - case OP_PHI: - case OP_FPHI: - case OP_VPHI: - case OP_XPHI: { - LLVMTypeRef phi_type = llvm_type_to_stack_type (type_to_llvm_type (ctx, &ins->klass->byval_arg)); + lcall = emit_call (ctx, bb, &builder, callee, args, LLVMCountParamTypes (llvm_sig)); - CHECK_FAILURE (ctx); +#ifdef LLVM_MONO_BRANCH + /* + * Modify cconv and parameter attributes to pass rgctx/imt correctly. + */ +#if defined(MONO_ARCH_IMT_REG) && defined(MONO_ARCH_RGCTX_REG) + g_assert (MONO_ARCH_IMT_REG == MONO_ARCH_RGCTX_REG); +#endif + /* The two can't be used together, so use only one LLVM calling conv to pass them */ + g_assert (!(call->rgctx_arg_reg && call->imt_arg_reg)); + if (!sig->pinvoke) + LLVMSetInstructionCallConv (lcall, LLVMMono1CallConv); + + if (call->rgctx_arg_reg) + LLVMAddInstrAttribute (lcall, 1 + sinfo.rgctx_arg_pindex, LLVMInRegAttribute); + if (call->imt_arg_reg) + LLVMAddInstrAttribute (lcall, 1 + sinfo.imt_arg_pindex, LLVMInRegAttribute); +#endif - if (ins->opcode == OP_VPHI) { - /* Treat valuetype PHI nodes as operating on the address itself */ - g_assert (ins->klass); - phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0); - } + /* Add byval attributes if needed */ + for (i = 0; i < sig->param_count; ++i) { + LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i + sig->hasthis] : NULL; - /* - * Have to precreate these, as they can be referenced by - * earlier instructions. - */ - sprintf (dname_buf, "t%d", ins->dreg); - dname = dname_buf; - values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname); + if (ainfo && ainfo->storage == LLVMArgVtypeByVal) { + LLVMAddInstrAttribute (lcall, 1 + sinfo.pindexes [i], LLVMByValAttribute); + } + } - if (ins->opcode == OP_VPHI) - addresses [ins->dreg] = values [ins->dreg]; + /* + * Convert the result + */ + if (cinfo && cinfo->ret.storage == LLVMArgVtypeInReg) { + LLVMValueRef regs [2]; - g_ptr_array_add (phi_values, values [ins->dreg]); + if (!addresses [ins->dreg]) + addresses [ins->dreg] = build_alloca (ctx, sig->ret); - /* - * Set the expected type of the incoming arguments since these have - * to have the same type. - */ - for (i = 0; i < ins->inst_phi_args [0]; i++) { - int sreg1 = ins->inst_phi_args [i + 1]; + regs [0] = LLVMBuildExtractValue (builder, lcall, 0, ""); + if (cinfo->ret.pair_storage [1] != LLVMArgNone) + regs [1] = LLVMBuildExtractValue (builder, lcall, 1, ""); - if (sreg1 != -1) - vreg_types [sreg1] = phi_type; - } - break; - } - default: - break; - } - } - } + emit_reg_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs); + } else if (sig->ret->type != MONO_TYPE_VOID && !vretaddr) { + /* If the method returns an unsigned value, need to zext it */ - /* - * Create an ordering for bblocks, use the depth first order first, then - * put the exception handling bblocks last. - */ - for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) { - bb = cfg->bblocks [bb_index]; - if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) { - g_ptr_array_add (bblock_list, bb); - bblocks [bb->block_num].added = TRUE; - } + values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret)); } - for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { - if (!bblocks [bb->block_num].added) - g_ptr_array_add (bblock_list, bb); - } + *builder_ref = ctx->builder; - /* - * Second pass: generate code. - */ - for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) { - MonoInst *ins; - LLVMBasicBlockRef cbb; - LLVMBuilderRef builder; - gboolean has_terminator; - LLVMValueRef v; - LLVMValueRef lhs, rhs; + g_free (sinfo.pindexes); + + return; + FAILURE: + return; +} - bb = g_ptr_array_index (bblock_list, bb_index); +static void +process_bb (EmitContext *ctx, MonoBasicBlock *bb) +{ + MonoCompile *cfg = ctx->cfg; + MonoMethodSignature *sig = ctx->sig; + LLVMValueRef method = ctx->lmethod; + LLVMValueRef *values = ctx->values; + LLVMValueRef *addresses = ctx->addresses; + int i; + LLVMCallInfo *linfo = ctx->linfo; + LLVMModuleRef module = ctx->module; + BBInfo *bblocks = ctx->bblocks; + MonoInst *ins; + LLVMBasicBlockRef cbb; + LLVMBuilderRef builder; + gboolean has_terminator; + LLVMValueRef v; + LLVMValueRef lhs, rhs; - if (!(bb == cfg->bb_entry || bb->in_count > 0)) - continue; + cbb = get_bb (ctx, bb); + builder = create_builder (ctx); + ctx->builder = builder; + LLVMPositionBuilderAtEnd (builder, cbb); - cbb = get_bb (ctx, bb); - builder = create_builder (ctx); - ctx->builder = builder; - LLVMPositionBuilderAtEnd (builder, cbb); + if (bb == cfg->bb_entry) + emit_entry_bb (ctx, builder); + CHECK_FAILURE (ctx); - if (bb == cfg->bb_entry) - emit_entry_bb (ctx, builder, pindexes); - CHECK_FAILURE (ctx); + if (bb->flags & BB_EXCEPTION_HANDLER) { + LLVMTypeRef i8ptr; + LLVMValueRef eh_selector, eh_exception, personality, args [4]; + LLVMBasicBlockRef target_bb; + MonoInst *exvar; + static gint32 mapping_inited; + static int ti_generator; + char ti_name [128]; + MonoClass **ti; + LLVMValueRef type_info; + int clause_index; + + if (!bblocks [bb->block_num].invoke_target) { + /* + * LLVM asserts if llvm.eh.selector is called from a bblock which + * doesn't have an invoke pointing at it. + */ + LLVM_FAILURE (ctx, "handler without invokes"); + } - if (bb->flags & BB_EXCEPTION_HANDLER) { - LLVMTypeRef i8ptr; - LLVMValueRef eh_selector, eh_exception, personality, args [4]; - MonoInst *exvar; - static gint32 mapping_inited; - static int ti_generator; - char ti_name [128]; - MonoClass **ti; - LLVMValueRef type_info; - int clause_index; - - if (!bblocks [bb->block_num].invoke_target) { - /* - * LLVM asserts if llvm.eh.selector is called from a bblock which - * doesn't have an invoke pointing at it. - */ - LLVM_FAILURE (ctx, "handler without invokes"); - } + eh_selector = LLVMGetNamedFunction (module, eh_selector_name); + + if (cfg->compile_aot) { + /* Use a dummy personality function */ + personality = LLVMGetNamedFunction (module, "mono_aot_personality"); + g_assert (personality); + } else { + personality = LLVMGetNamedFunction (module, "mono_personality"); + if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0) + LLVMAddGlobalMapping (ee, personality, mono_personality); + } - eh_selector = LLVMGetNamedFunction (module, "llvm.eh.selector"); + i8ptr = LLVMPointerType (LLVMInt8Type (), 0); - if (cfg->compile_aot) { - /* Use a dummy personality function */ - personality = LLVMGetNamedFunction (module, "mono_aot_personality"); - g_assert (personality); - } else { - personality = LLVMGetNamedFunction (module, "mono_personality"); - if (InterlockedCompareExchange (&mapping_inited, 1, 0) == 0) - LLVMAddGlobalMapping (ee, personality, mono_personality); - } + clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1; + + /* + * Create the type info + */ + sprintf (ti_name, "type_info_%d", ti_generator); + ti_generator ++; - i8ptr = LLVMPointerType (LLVMInt8Type (), 0); + if (cfg->compile_aot) { + /* decode_eh_frame () in aot-runtime.c will decode this */ + type_info = LLVMAddGlobal (module, LLVMInt32Type (), ti_name); + LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE)); - clause_index = (mono_get_block_region_notry (cfg, bb->region) >> 8) - 1; + LLVMSetLinkage (type_info, LLVMPrivateLinkage); + LLVMSetVisibility (type_info, LLVMHiddenVisibility); + /* + * Enabling this causes llc to crash: + * http://llvm.org/bugs/show_bug.cgi?id=6102 + */ + //LLVM_FAILURE (ctx, "aot+clauses"); + } else { /* - * Create the type info + * After the cfg mempool is freed, the type info will point to stale memory, + * but this is not a problem, since we decode it once in exception_cb during + * compilation. */ - sprintf (ti_name, "type_info_%d", ti_generator); - ti_generator ++; + ti = mono_mempool_alloc (cfg->mempool, sizeof (gint32)); + *(gint32*)ti = clause_index; - if (cfg->compile_aot) { - /* decode_eh_frame () in aot-runtime.c will decode this */ - type_info = LLVMAddGlobal (module, LLVMInt32Type (), ti_name); - LLVMSetInitializer (type_info, LLVMConstInt (LLVMInt32Type (), clause_index, FALSE)); + type_info = LLVMAddGlobal (module, i8ptr, ti_name); - /* - * FIXME: llc currently generates incorrect data in the LSDA: - * .byte 0x9B # @TType format (indirect pcrel sdata4) - * and later: - * .quad type_info_1 # TypeInfo - */ - LLVM_FAILURE (ctx, "aot+clauses"); - } else { - /* exception_cb will decode this */ - ti = g_malloc (sizeof (MonoExceptionClause)); - memcpy (ti, &mono_method_get_header (cfg->method)->clauses [clause_index], sizeof (MonoExceptionClause)); + LLVMAddGlobalMapping (ee, type_info, ti); + } - type_info = LLVMAddGlobal (module, i8ptr, ti_name); + args [0] = LLVMConstNull (i8ptr); + args [1] = LLVMConstBitCast (personality, i8ptr); + args [2] = type_info; + LLVMBuildCall (builder, eh_selector, args, 3, ""); - LLVMAddGlobalMapping (ee, type_info, ti); - } + /* Store the exception into the exvar */ + if (bb->in_scount == 1) { + g_assert (bb->in_scount == 1); + exvar = bb->in_stack [0]; + + eh_exception = LLVMGetNamedFunction (module, "llvm.eh.exception"); - args [0] = LLVMConstNull (i8ptr); - args [1] = LLVMConstBitCast (personality, i8ptr); - args [2] = type_info; - LLVMBuildCall (builder, eh_selector, args, 3, ""); + // FIXME: This is shared with filter clauses ? + g_assert (!values [exvar->dreg]); + values [exvar->dreg] = LLVMBuildCall (builder, eh_exception, NULL, 0, ""); + emit_volatile_store (ctx, exvar->dreg); + } - /* Store the exception into the exvar */ - if (bb->in_scount == 1) { - g_assert (bb->in_scount == 1); - exvar = bb->in_stack [0]; + /* Start a new bblock which CALL_HANDLER can branch to */ + target_bb = bblocks [bb->block_num].call_handler_target_bb; + if (target_bb) { + LLVMBuildBr (builder, target_bb); - eh_exception = LLVMGetNamedFunction (module, "llvm.eh.exception"); + ctx->builder = builder = create_builder (ctx); + LLVMPositionBuilderAtEnd (ctx->builder, target_bb); - // FIXME: This is shared with filter clauses ? - g_assert (!values [exvar->dreg]); - values [exvar->dreg] = LLVMBuildCall (builder, eh_exception, NULL, 0, ""); - emit_volatile_store (ctx, exvar->dreg); - } + ctx->bblocks [bb->block_num].end_bblock = target_bb; } + } - has_terminator = FALSE; - for (ins = bb->code; ins; ins = ins->next) { - const char *spec = LLVM_INS_INFO (ins->opcode); - char *dname = NULL; - char dname_buf [128]; + has_terminator = FALSE; + for (ins = bb->code; ins; ins = ins->next) { + const char *spec = LLVM_INS_INFO (ins->opcode); + char *dname = NULL; + char dname_buf [128]; - if (has_terminator) - /* There could be instructions after a terminator, skip them */ - break; + if (has_terminator) + /* There could be instructions after a terminator, skip them */ + break; - if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) { - sprintf (dname_buf, "t%d", ins->dreg); - dname = dname_buf; - } + if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins)) { + sprintf (dname_buf, "t%d", ins->dreg); + dname = dname_buf; + } - if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') { - MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1); + if (spec [MONO_INST_SRC1] != ' ' && spec [MONO_INST_SRC1] != 'v') { + MonoInst *var = get_vreg_to_inst (cfg, ins->sreg1); - if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) { - lhs = emit_volatile_load (ctx, ins->sreg1); - } else { - /* It is ok for SETRET to have an uninitialized argument */ - if (!values [ins->sreg1] && ins->opcode != OP_SETRET) - LLVM_FAILURE (ctx, "sreg1"); - lhs = values [ins->sreg1]; - } + if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) { + lhs = emit_volatile_load (ctx, ins->sreg1); } else { - lhs = NULL; + /* It is ok for SETRET to have an uninitialized argument */ + if (!values [ins->sreg1] && ins->opcode != OP_SETRET) + LLVM_FAILURE (ctx, "sreg1"); + lhs = values [ins->sreg1]; } + } else { + lhs = NULL; + } - if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') { - MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2); - if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) { - rhs = emit_volatile_load (ctx, ins->sreg2); - } else { - if (!values [ins->sreg2]) - LLVM_FAILURE (ctx, "sreg2"); - rhs = values [ins->sreg2]; - } + if (spec [MONO_INST_SRC2] != ' ' && spec [MONO_INST_SRC2] != ' ') { + MonoInst *var = get_vreg_to_inst (cfg, ins->sreg2); + if (var && var->flags & (MONO_INST_VOLATILE|MONO_INST_INDIRECT)) { + rhs = emit_volatile_load (ctx, ins->sreg2); } else { - rhs = NULL; + if (!values [ins->sreg2]) + LLVM_FAILURE (ctx, "sreg2"); + rhs = values [ins->sreg2]; } + } else { + rhs = NULL; + } - //mono_print_ins (ins); - switch (ins->opcode) { - case OP_NOP: - case OP_NOT_NULL: - case OP_LIVERANGE_START: - case OP_LIVERANGE_END: - break; - case OP_ICONST: - values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE); - break; - case OP_I8CONST: + //mono_print_ins (ins); + switch (ins->opcode) { + case OP_NOP: + case OP_NOT_NULL: + case OP_LIVERANGE_START: + case OP_LIVERANGE_END: + break; + case OP_ICONST: + values [ins->dreg] = LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE); + break; + case OP_I8CONST: #if SIZEOF_VOID_P == 4 - values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE); + values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE); #else - values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE); + values [ins->dreg] = LLVMConstInt (LLVMInt64Type (), (gint64)ins->inst_c0, FALSE); #endif - break; - case OP_R8CONST: - values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0); - break; - case OP_R4CONST: - values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ()); - break; - case OP_BR: - LLVMBuildBr (builder, get_bb (ctx, ins->inst_target_bb)); - has_terminator = TRUE; - break; - case OP_SWITCH: { - int i; - LLVMValueRef v; - char bb_name [128]; - LLVMBasicBlockRef new_bb; - LLVMBuilderRef new_builder; - - // The default branch is already handled - // FIXME: Handle it here - - /* Start new bblock */ - sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++); - new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name); - - lhs = convert (ctx, lhs, LLVMInt32Type ()); - v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass)); - for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) { - MonoBasicBlock *target_bb = ins->inst_many_bb [i]; - - LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb)); - } + break; + case OP_R8CONST: + values [ins->dreg] = LLVMConstReal (LLVMDoubleType (), *(double*)ins->inst_p0); + break; + case OP_R4CONST: + values [ins->dreg] = LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), *(float*)ins->inst_p0), LLVMDoubleType ()); + break; + case OP_BR: + LLVMBuildBr (builder, get_bb (ctx, ins->inst_target_bb)); + has_terminator = TRUE; + break; + case OP_SWITCH: { + int i; + LLVMValueRef v; + char bb_name [128]; + LLVMBasicBlockRef new_bb; + LLVMBuilderRef new_builder; + + // The default branch is already handled + // FIXME: Handle it here + + /* Start new bblock */ + sprintf (bb_name, "SWITCH_DEFAULT_BB%d", ctx->default_index ++); + new_bb = LLVMAppendBasicBlock (ctx->lmethod, bb_name); + + lhs = convert (ctx, lhs, LLVMInt32Type ()); + v = LLVMBuildSwitch (builder, lhs, new_bb, GPOINTER_TO_UINT (ins->klass)); + for (i = 0; i < GPOINTER_TO_UINT (ins->klass); ++i) { + MonoBasicBlock *target_bb = ins->inst_many_bb [i]; + + LLVMAddCase (v, LLVMConstInt (LLVMInt32Type (), i, FALSE), get_bb (ctx, target_bb)); + } - new_builder = create_builder (ctx); - LLVMPositionBuilderAtEnd (new_builder, new_bb); - LLVMBuildUnreachable (new_builder); + new_builder = create_builder (ctx); + LLVMPositionBuilderAtEnd (new_builder, new_bb); + LLVMBuildUnreachable (new_builder); - has_terminator = TRUE; - g_assert (!ins->next); + has_terminator = TRUE; + g_assert (!ins->next); - break; - } + break; + } - case OP_SETRET: - if (linfo->ret.storage == LLVMArgVtypeInReg) { - LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method))); - LLVMValueRef part1, retval; - int size; + case OP_SETRET: + if (linfo->ret.storage == LLVMArgVtypeInReg) { + LLVMTypeRef ret_type = LLVMGetReturnType (LLVMGetElementType (LLVMTypeOf (method))); + LLVMValueRef part1, retval; + int size; - size = get_vtype_size (sig->ret); + size = get_vtype_size (sig->ret); - g_assert (addresses [ins->sreg1]); + g_assert (addresses [ins->sreg1]); - g_assert (linfo->ret.pair_storage [0] == LLVMArgInIReg); - g_assert (linfo->ret.pair_storage [1] == LLVMArgNone); + g_assert (linfo->ret.pair_storage [0] == LLVMArgInIReg); + g_assert (linfo->ret.pair_storage [1] == LLVMArgNone); - part1 = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMIntType (size * 8), 0), ""), ""), IntPtrType ()); + part1 = convert (ctx, LLVMBuildLoad (builder, LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMIntType (size * 8), 0), ""), ""), IntPtrType ()); - retval = LLVMBuildInsertValue (builder, LLVMGetUndef (ret_type), part1, 0, ""); + retval = LLVMBuildInsertValue (builder, LLVMGetUndef (ret_type), part1, 0, ""); - LLVMBuildRet (builder, retval); - break; - } + LLVMBuildRet (builder, retval); + break; + } - if (linfo->ret.storage == LLVMArgVtypeRetAddr) { - LLVMBuildRetVoid (builder); - break; - } + if (linfo->ret.storage == LLVMArgVtypeRetAddr) { + LLVMBuildRetVoid (builder); + break; + } - if (!lhs || is_dead [ins->sreg1]) { - /* - * The method did not set its return value, probably because it - * ends with a throw. - */ - if (cfg->vret_addr) - LLVMBuildRetVoid (builder); - else - LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret))); - } else { - LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret))); - } - has_terminator = TRUE; + if (!lhs || ctx->is_dead [ins->sreg1]) { + /* + * The method did not set its return value, probably because it + * ends with a throw. + */ + if (cfg->vret_addr) + LLVMBuildRetVoid (builder); + else + LLVMBuildRet (builder, LLVMConstNull (type_to_llvm_type (ctx, sig->ret))); + } else { + LLVMBuildRet (builder, convert (ctx, lhs, type_to_llvm_type (ctx, sig->ret))); + } + has_terminator = TRUE; + break; + case OP_ICOMPARE: + case OP_FCOMPARE: + case OP_LCOMPARE: + case OP_COMPARE: + case OP_ICOMPARE_IMM: + case OP_LCOMPARE_IMM: + case OP_COMPARE_IMM: { + CompRelation rel; + LLVMValueRef cmp; + + if (ins->next->opcode == OP_NOP) break; - case OP_ICOMPARE: - case OP_FCOMPARE: - case OP_LCOMPARE: - case OP_COMPARE: - case OP_ICOMPARE_IMM: - case OP_LCOMPARE_IMM: - case OP_COMPARE_IMM: -#ifdef TARGET_AMD64 - case OP_AMD64_ICOMPARE_MEMBASE_REG: - case OP_AMD64_ICOMPARE_MEMBASE_IMM: -#endif -#ifdef TARGET_X86 - case OP_X86_COMPARE_MEMBASE_REG: - case OP_X86_COMPARE_MEMBASE_IMM: -#endif - { - CompRelation rel; - LLVMValueRef cmp; - if (ins->next->opcode == OP_NOP) - break; - - if (ins->next->opcode == OP_BR) - /* The comparison result is not needed */ - continue; - - rel = mono_opcode_to_cond (ins->next->opcode); - - /* Used for implementing bound checks */ -#ifdef TARGET_AMD64 - if ((ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_REG) || (ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_IMM)) { - int size = 4; - LLVMValueRef index; - LLVMTypeRef t; - - t = LLVMInt32Type (); - - g_assert (ins->inst_offset % size == 0); - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); - - lhs = LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, ""), ""); - } - if (ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_IMM) { - lhs = convert (ctx, lhs, LLVMInt32Type ()); - rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); - } - if (ins->opcode == OP_AMD64_ICOMPARE_MEMBASE_REG) - rhs = convert (ctx, rhs, LLVMInt32Type ()); -#endif + if (ins->next->opcode == OP_BR) + /* The comparison result is not needed */ + continue; -#ifdef TARGET_X86 - if ((ins->opcode == OP_X86_COMPARE_MEMBASE_REG) || (ins->opcode == OP_X86_COMPARE_MEMBASE_IMM)) { - int size = 4; - LLVMValueRef index; - LLVMTypeRef t; + rel = mono_opcode_to_cond (ins->next->opcode); - t = LLVMInt32Type (); + if (ins->opcode == OP_ICOMPARE_IMM) { + lhs = convert (ctx, lhs, LLVMInt32Type ()); + rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); + } + if (ins->opcode == OP_LCOMPARE_IMM) { + lhs = convert (ctx, lhs, LLVMInt64Type ()); + rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE); + } + if (ins->opcode == OP_LCOMPARE) { + lhs = convert (ctx, lhs, LLVMInt64Type ()); + rhs = convert (ctx, rhs, LLVMInt64Type ()); + } + if (ins->opcode == OP_ICOMPARE) { + lhs = convert (ctx, lhs, LLVMInt32Type ()); + rhs = convert (ctx, rhs, LLVMInt32Type ()); + } - g_assert (ins->inst_offset % size == 0); - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); + if (lhs && rhs) { + if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind) + rhs = convert (ctx, rhs, LLVMTypeOf (lhs)); + else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind) + lhs = convert (ctx, lhs, LLVMTypeOf (rhs)); + } - lhs = LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, ""), ""); - } - if (ins->opcode == OP_X86_COMPARE_MEMBASE_IMM) { - lhs = convert (ctx, lhs, LLVMInt32Type ()); - rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); + /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */ + if (ins->opcode == OP_FCOMPARE) + cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), ""); + else if (ins->opcode == OP_COMPARE_IMM) + cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), ""); + else if (ins->opcode == OP_LCOMPARE_IMM) { + if (SIZEOF_REGISTER == 4 && COMPILE_LLVM (cfg)) { + /* The immediate is encoded in two fields */ + guint64 l = ((guint64)(guint32)ins->inst_offset << 32) | ((guint32)ins->inst_imm); + cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), l, FALSE), ""); + } else { + cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, LLVMInt64Type ()), LLVMConstInt (LLVMInt64Type (), ins->inst_imm, FALSE), ""); } - if (ins->opcode == OP_X86_COMPARE_MEMBASE_REG) - rhs = convert (ctx, rhs, LLVMInt32Type ()); -#endif + } + else if (ins->opcode == OP_COMPARE) + cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), ""); + else + cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, ""); - if (ins->opcode == OP_ICOMPARE_IMM) { - lhs = convert (ctx, lhs, LLVMInt32Type ()); - rhs = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); - } - if (ins->opcode == OP_LCOMPARE_IMM) { - lhs = convert (ctx, lhs, LLVMInt64Type ()); - rhs = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE); - } - if (ins->opcode == OP_LCOMPARE) { - lhs = convert (ctx, lhs, LLVMInt64Type ()); - rhs = convert (ctx, rhs, LLVMInt64Type ()); - } - if (ins->opcode == OP_ICOMPARE) { - lhs = convert (ctx, lhs, LLVMInt32Type ()); - rhs = convert (ctx, rhs, LLVMInt32Type ()); - } + if (MONO_IS_COND_BRANCH_OP (ins->next)) { + LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb)); + has_terminator = TRUE; + } else if (MONO_IS_SETCC (ins->next)) { + sprintf (dname_buf, "t%d", ins->next->dreg); + dname = dname_buf; + values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname); - if (lhs && rhs) { - if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind) - rhs = convert (ctx, rhs, LLVMTypeOf (lhs)); - else if (LLVMGetTypeKind (LLVMTypeOf (rhs)) == LLVMPointerTypeKind) - lhs = convert (ctx, lhs, LLVMTypeOf (rhs)); - } + /* Add stores for volatile variables */ + emit_volatile_store (ctx, ins->next->dreg); + } else if (MONO_IS_COND_EXC (ins->next)) { + emit_cond_system_exception (ctx, bb, ins->next->inst_p1, cmp); + CHECK_FAILURE (ctx); + builder = ctx->builder; + } else { + LLVM_FAILURE (ctx, "next"); + } - /* We use COMPARE+SETcc/Bcc, llvm uses SETcc+br cond */ - if (ins->opcode == OP_FCOMPARE) - cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), ""); - else if (ins->opcode == OP_COMPARE_IMM) - cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), ""); - else if (ins->opcode == OP_COMPARE) - cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], convert (ctx, lhs, IntPtrType ()), convert (ctx, rhs, IntPtrType ()), ""); + ins = ins->next; + break; + } + case OP_FCEQ: + case OP_FCLT: + case OP_FCLT_UN: + case OP_FCGT: + case OP_FCGT_UN: { + CompRelation rel; + LLVMValueRef cmp; + + rel = mono_opcode_to_cond (ins->opcode); + + cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), ""); + values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname); + break; + } + case OP_PHI: + case OP_FPHI: + case OP_VPHI: + case OP_XPHI: { + int i; + gboolean empty = TRUE; + + /* Check that all input bblocks really branch to us */ + for (i = 0; i < bb->in_count; ++i) { + if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED) + ins->inst_phi_args [i + 1] = -1; else - cmp = LLVMBuildICmp (builder, cond_to_llvm_cond [rel], lhs, rhs, ""); - - if (MONO_IS_COND_BRANCH_OP (ins->next)) { - LLVMBuildCondBr (builder, cmp, get_bb (ctx, ins->next->inst_true_bb), get_bb (ctx, ins->next->inst_false_bb)); - has_terminator = TRUE; - } else if (MONO_IS_SETCC (ins->next)) { - sprintf (dname_buf, "t%d", ins->next->dreg); - dname = dname_buf; - values [ins->next->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname); - - /* Add stores for volatile variables */ - emit_volatile_store (ctx, ins->next->dreg); - } else if (MONO_IS_COND_EXC (ins->next)) { - //emit_cond_throw_pos (ctx); - emit_cond_system_exception (ctx, bb, ins->next->inst_p1, cmp); - builder = ctx->builder; - } else { - LLVM_FAILURE (ctx, "next"); - } - - ins = ins->next; - break; + empty = FALSE; } - case OP_FCEQ: - case OP_FCLT: - case OP_FCLT_UN: - case OP_FCGT: - case OP_FCGT_UN: { - CompRelation rel; - LLVMValueRef cmp; - rel = mono_opcode_to_cond (ins->opcode); - - cmp = LLVMBuildFCmp (builder, fpcond_to_llvm_cond [rel], convert (ctx, lhs, LLVMDoubleType ()), convert (ctx, rhs, LLVMDoubleType ()), ""); - values [ins->dreg] = LLVMBuildZExt (builder, cmp, LLVMInt32Type (), dname); + if (empty) { + /* LLVM doesn't like phi instructions with zero operands */ + ctx->is_dead [ins->dreg] = TRUE; break; - } - case OP_PHI: - case OP_FPHI: - case OP_VPHI: - case OP_XPHI: { - int i; - gboolean empty = TRUE; - - /* Check that all input bblocks really branch to us */ - for (i = 0; i < bb->in_count; ++i) { - if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_NOT_REACHED) - ins->inst_phi_args [i + 1] = -1; - else - empty = FALSE; - } + } - if (empty) { - /* LLVM doesn't like phi instructions with zero operands */ - is_dead [ins->dreg] = TRUE; - break; - } + /* Created earlier, insert it now */ + LLVMInsertIntoBuilder (builder, values [ins->dreg]); - /* Created earlier, insert it now */ - LLVMInsertIntoBuilder (builder, values [ins->dreg]); + for (i = 0; i < ins->inst_phi_args [0]; i++) { + int sreg1 = ins->inst_phi_args [i + 1]; + int count, j; - for (i = 0; i < ins->inst_phi_args [0]; i++) { - int sreg1 = ins->inst_phi_args [i + 1]; - int count, j; + /* + * Count the number of times the incoming bblock branches to us, + * since llvm requires a separate entry for each. + */ + if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) { + MonoInst *switch_ins = bb->in_bb [i]->last_ins; - /* - * Count the number of times the incoming bblock branches to us, - * since llvm requires a separate entry for each. - */ - if (bb->in_bb [i]->last_ins && bb->in_bb [i]->last_ins->opcode == OP_SWITCH) { - MonoInst *switch_ins = bb->in_bb [i]->last_ins; - - count = 0; - for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) { - if (switch_ins->inst_many_bb [j] == bb) - count ++; - } - } else { - count = 1; + count = 0; + for (j = 0; j < GPOINTER_TO_UINT (switch_ins->klass); ++j) { + if (switch_ins->inst_many_bb [j] == bb) + count ++; } + } else { + count = 1; + } - /* Remember for later */ - for (j = 0; j < count; ++j) { - PhiNode *node = mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode)); - node->bb = bb; - node->phi = ins; - node->in_bb = bb->in_bb [i]; - node->sreg = sreg1; - bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node); - } + /* Remember for later */ + for (j = 0; j < count; ++j) { + PhiNode *node = mono_mempool_alloc0 (ctx->mempool, sizeof (PhiNode)); + node->bb = bb; + node->phi = ins; + node->in_bb = bb->in_bb [i]; + node->sreg = sreg1; + bblocks [bb->in_bb [i]->block_num].phi_nodes = g_slist_prepend_mempool (ctx->mempool, bblocks [bb->in_bb [i]->block_num].phi_nodes, node); } - break; } - case OP_MOVE: - case OP_LMOVE: - case OP_XMOVE: - g_assert (lhs); - values [ins->dreg] = lhs; - break; - case OP_FMOVE: { - MonoInst *var = get_vreg_to_inst (cfg, ins->dreg); + break; + } + case OP_MOVE: + case OP_LMOVE: + case OP_XMOVE: + case OP_SETFRET: + g_assert (lhs); + values [ins->dreg] = lhs; + break; + case OP_FMOVE: { + MonoInst *var = get_vreg_to_inst (cfg, ins->dreg); - g_assert (lhs); - values [ins->dreg] = lhs; + g_assert (lhs); + values [ins->dreg] = lhs; - if (var && var->klass->byval_arg.type == MONO_TYPE_R4) { - /* - * This is added by the spilling pass in case of the JIT, - * but we have to do it ourselves. - */ - values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ()); - } - break; + if (var && var->klass->byval_arg.type == MONO_TYPE_R4) { + /* + * This is added by the spilling pass in case of the JIT, + * but we have to do it ourselves. + */ + values [ins->dreg] = convert (ctx, values [ins->dreg], LLVMFloatType ()); } + break; + } + case OP_IADD: + case OP_ISUB: + case OP_IAND: + case OP_IMUL: + case OP_IDIV: + case OP_IDIV_UN: + case OP_IREM: + case OP_IREM_UN: + case OP_IOR: + case OP_IXOR: + case OP_ISHL: + case OP_ISHR: + case OP_ISHR_UN: + case OP_FADD: + case OP_FSUB: + case OP_FMUL: + case OP_FDIV: + case OP_LADD: + case OP_LSUB: + case OP_LMUL: + case OP_LDIV: + case OP_LDIV_UN: + case OP_LREM: + case OP_LREM_UN: + case OP_LAND: + case OP_LOR: + case OP_LXOR: + case OP_LSHL: + case OP_LSHR: + case OP_LSHR_UN: + lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST])); + rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST])); + + switch (ins->opcode) { case OP_IADD: + case OP_LADD: + values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname); + break; case OP_ISUB: - case OP_IAND: + case OP_LSUB: + values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname); + break; case OP_IMUL: - case OP_IDIV: - case OP_IDIV_UN: + case OP_LMUL: + values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname); + break; case OP_IREM: + case OP_LREM: + values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname); + break; case OP_IREM_UN: - case OP_IOR: - case OP_IXOR: - case OP_ISHL: - case OP_ISHR: - case OP_ISHR_UN: - case OP_FADD: - case OP_FSUB: - case OP_FMUL: - case OP_FDIV: - case OP_LADD: - case OP_LSUB: - case OP_LMUL: + case OP_LREM_UN: + values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname); + break; + case OP_IDIV: case OP_LDIV: + values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname); + break; + case OP_IDIV_UN: case OP_LDIV_UN: - case OP_LREM: - case OP_LREM_UN: + values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname); + break; + case OP_FDIV: + values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname); + break; + case OP_IAND: case OP_LAND: + values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname); + break; + case OP_IOR: case OP_LOR: + values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname); + break; + case OP_IXOR: case OP_LXOR: + values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname); + break; + case OP_ISHL: case OP_LSHL: + values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname); + break; + case OP_ISHR: case OP_LSHR: + values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname); + break; + case OP_ISHR_UN: case OP_LSHR_UN: - lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST])); - rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST])); - - switch (ins->opcode) { - case OP_IADD: - case OP_FADD: - case OP_LADD: - values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, dname); - break; - case OP_ISUB: - case OP_FSUB: - case OP_LSUB: - values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, dname); - break; - case OP_IMUL: - case OP_FMUL: - case OP_LMUL: - values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, dname); - break; - case OP_IREM: - case OP_LREM: - values [ins->dreg] = LLVMBuildSRem (builder, lhs, rhs, dname); - break; - case OP_IREM_UN: - case OP_LREM_UN: - values [ins->dreg] = LLVMBuildURem (builder, lhs, rhs, dname); - break; - case OP_IDIV: - case OP_LDIV: - values [ins->dreg] = LLVMBuildSDiv (builder, lhs, rhs, dname); - break; - case OP_IDIV_UN: - case OP_LDIV_UN: - values [ins->dreg] = LLVMBuildUDiv (builder, lhs, rhs, dname); - break; - case OP_FDIV: - values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, dname); - break; - case OP_IAND: - case OP_LAND: - values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, dname); - break; - case OP_IOR: - case OP_LOR: - values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, dname); - break; - case OP_IXOR: - case OP_LXOR: - values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, dname); - break; - case OP_ISHL: - case OP_LSHL: - values [ins->dreg] = LLVMBuildShl (builder, lhs, rhs, dname); - break; - case OP_ISHR: - case OP_LSHR: - values [ins->dreg] = LLVMBuildAShr (builder, lhs, rhs, dname); - break; - case OP_ISHR_UN: - case OP_LSHR_UN: - values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname); - break; - default: - g_assert_not_reached (); - } + values [ins->dreg] = LLVMBuildLShr (builder, lhs, rhs, dname); break; - case OP_IADD_IMM: - case OP_ISUB_IMM: - case OP_IMUL_IMM: - case OP_IREM_IMM: - case OP_IREM_UN_IMM: - case OP_IDIV_IMM: - case OP_IDIV_UN_IMM: - case OP_IAND_IMM: - case OP_IOR_IMM: - case OP_IXOR_IMM: - case OP_ISHL_IMM: - case OP_ISHR_IMM: - case OP_ISHR_UN_IMM: - case OP_LADD_IMM: - case OP_LSUB_IMM: - case OP_LREM_IMM: - case OP_LAND_IMM: - case OP_LOR_IMM: - case OP_LXOR_IMM: - case OP_LSHL_IMM: - case OP_LSHR_IMM: - case OP_LSHR_UN_IMM: - case OP_ADD_IMM: - case OP_AND_IMM: - case OP_MUL_IMM: - case OP_SHL_IMM: - case OP_SHR_IMM: { - LLVMValueRef imm; - - if (spec [MONO_INST_SRC1] == 'l') { - imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE); - } else { - imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); - } - -#if SIZEOF_VOID_P == 4 - if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM) - imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); -#endif - if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind) - lhs = convert (ctx, lhs, IntPtrType ()); - imm = convert (ctx, imm, LLVMTypeOf (lhs)); - switch (ins->opcode) { - case OP_IADD_IMM: - case OP_LADD_IMM: - case OP_ADD_IMM: - values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname); - break; - case OP_ISUB_IMM: - case OP_LSUB_IMM: - values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname); - break; - case OP_IMUL_IMM: - case OP_MUL_IMM: - values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname); - break; - case OP_IDIV_IMM: - case OP_LDIV_IMM: - values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname); - break; - case OP_IDIV_UN_IMM: - case OP_LDIV_UN_IMM: - values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname); - break; - case OP_IREM_IMM: - case OP_LREM_IMM: - values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname); - break; - case OP_IREM_UN_IMM: - values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname); - break; - case OP_IAND_IMM: - case OP_LAND_IMM: - case OP_AND_IMM: - values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname); - break; - case OP_IOR_IMM: - case OP_LOR_IMM: - values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname); - break; - case OP_IXOR_IMM: - case OP_LXOR_IMM: - values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname); - break; - case OP_ISHL_IMM: - case OP_LSHL_IMM: - case OP_SHL_IMM: - values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname); - break; - case OP_ISHR_IMM: - case OP_LSHR_IMM: - case OP_SHR_IMM: - values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname); - break; - case OP_ISHR_UN_IMM: - /* This is used to implement conv.u4, so the lhs could be an i8 */ - lhs = convert (ctx, lhs, LLVMInt32Type ()); - imm = convert (ctx, imm, LLVMInt32Type ()); - values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname); - break; - case OP_LSHR_UN_IMM: - values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname); - break; - default: - g_assert_not_reached (); - } - break; - } - case OP_INEG: - values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname); - break; - case OP_LNEG: - values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname); + case OP_FADD: + values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, dname); break; - case OP_FNEG: - lhs = convert (ctx, lhs, LLVMDoubleType ()); - values [ins->dreg] = LLVMBuildSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname); + case OP_FSUB: + values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, dname); break; - case OP_INOT: { - guint32 v = 0xffffffff; - values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), lhs, dname); + case OP_FMUL: + values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, dname); break; + + default: + g_assert_not_reached (); } - case OP_LNOT: { - guint64 v = 0xffffffffffffffffLL; - values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname); - break; + break; + case OP_IADD_IMM: + case OP_ISUB_IMM: + case OP_IMUL_IMM: + case OP_IREM_IMM: + case OP_IREM_UN_IMM: + case OP_IDIV_IMM: + case OP_IDIV_UN_IMM: + case OP_IAND_IMM: + case OP_IOR_IMM: + case OP_IXOR_IMM: + case OP_ISHL_IMM: + case OP_ISHR_IMM: + case OP_ISHR_UN_IMM: + case OP_LADD_IMM: + case OP_LSUB_IMM: + case OP_LREM_IMM: + case OP_LAND_IMM: + case OP_LOR_IMM: + case OP_LXOR_IMM: + case OP_LSHL_IMM: + case OP_LSHR_IMM: + case OP_LSHR_UN_IMM: + case OP_ADD_IMM: + case OP_AND_IMM: + case OP_MUL_IMM: + case OP_SHL_IMM: + case OP_SHR_IMM: { + LLVMValueRef imm; + + if (spec [MONO_INST_SRC1] == 'l') { + imm = LLVMConstInt (LLVMInt64Type (), GET_LONG_IMM (ins), FALSE); + } else { + imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); } -#if defined(TARGET_X86) || defined(TARGET_AMD64) - case OP_X86_LEA: { - LLVMValueRef v1, v2; - v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), ""); - v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, ""); - values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname); - break; - } +#if SIZEOF_VOID_P == 4 + if (ins->opcode == OP_LSHL_IMM || ins->opcode == OP_LSHR_IMM || ins->opcode == OP_LSHR_UN_IMM) + imm = LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE); #endif - case OP_ICONV_TO_I1: - case OP_ICONV_TO_I2: - case OP_ICONV_TO_I4: - case OP_ICONV_TO_U1: - case OP_ICONV_TO_U2: - case OP_ICONV_TO_U4: - case OP_LCONV_TO_I1: - case OP_LCONV_TO_I2: - case OP_LCONV_TO_U1: - case OP_LCONV_TO_U2: - case OP_LCONV_TO_U4: { - gboolean sign; - - sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2); - - /* Have to do two casts since our vregs have type int */ - v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), ""); - if (sign) - values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname); - else - values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname); - break; - } - case OP_ICONV_TO_I8: - values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname); - break; - case OP_ICONV_TO_U8: - values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname); - break; - case OP_FCONV_TO_I4: - values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname); + if (LLVMGetTypeKind (LLVMTypeOf (lhs)) == LLVMPointerTypeKind) + lhs = convert (ctx, lhs, IntPtrType ()); + imm = convert (ctx, imm, LLVMTypeOf (lhs)); + switch (ins->opcode) { + case OP_IADD_IMM: + case OP_LADD_IMM: + case OP_ADD_IMM: + values [ins->dreg] = LLVMBuildAdd (builder, lhs, imm, dname); break; - case OP_FCONV_TO_I1: - values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), ""); + case OP_ISUB_IMM: + case OP_LSUB_IMM: + values [ins->dreg] = LLVMBuildSub (builder, lhs, imm, dname); break; - case OP_FCONV_TO_U1: - values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), ""); + case OP_IMUL_IMM: + case OP_MUL_IMM: + values [ins->dreg] = LLVMBuildMul (builder, lhs, imm, dname); break; - case OP_FCONV_TO_I2: - values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), ""); + case OP_IDIV_IMM: + case OP_LDIV_IMM: + values [ins->dreg] = LLVMBuildSDiv (builder, lhs, imm, dname); break; - case OP_FCONV_TO_U2: - values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), ""); + case OP_IDIV_UN_IMM: + case OP_LDIV_UN_IMM: + values [ins->dreg] = LLVMBuildUDiv (builder, lhs, imm, dname); break; - case OP_FCONV_TO_I8: - values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname); + case OP_IREM_IMM: + case OP_LREM_IMM: + values [ins->dreg] = LLVMBuildSRem (builder, lhs, imm, dname); break; - case OP_FCONV_TO_I: - values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname); + case OP_IREM_UN_IMM: + values [ins->dreg] = LLVMBuildURem (builder, lhs, imm, dname); break; - case OP_ICONV_TO_R8: - case OP_LCONV_TO_R8: - values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname); + case OP_IAND_IMM: + case OP_LAND_IMM: + case OP_AND_IMM: + values [ins->dreg] = LLVMBuildAnd (builder, lhs, imm, dname); break; - case OP_LCONV_TO_R_UN: - values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname); + case OP_IOR_IMM: + case OP_LOR_IMM: + values [ins->dreg] = LLVMBuildOr (builder, lhs, imm, dname); break; -#if SIZEOF_VOID_P == 4 - case OP_LCONV_TO_U: -#endif - case OP_LCONV_TO_I4: - values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname); + case OP_IXOR_IMM: + case OP_LXOR_IMM: + values [ins->dreg] = LLVMBuildXor (builder, lhs, imm, dname); break; - case OP_ICONV_TO_R4: - case OP_LCONV_TO_R4: - v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), ""); - values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname); + case OP_ISHL_IMM: + case OP_LSHL_IMM: + case OP_SHL_IMM: + values [ins->dreg] = LLVMBuildShl (builder, lhs, imm, dname); break; - case OP_FCONV_TO_R4: - v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""); - values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname); + case OP_ISHR_IMM: + case OP_LSHR_IMM: + case OP_SHR_IMM: + values [ins->dreg] = LLVMBuildAShr (builder, lhs, imm, dname); break; - case OP_SEXT_I4: - values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname); + case OP_ISHR_UN_IMM: + /* This is used to implement conv.u4, so the lhs could be an i8 */ + lhs = convert (ctx, lhs, LLVMInt32Type ()); + imm = convert (ctx, imm, LLVMInt32Type ()); + values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname); break; - case OP_ZEXT_I4: - values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname); + case OP_LSHR_UN_IMM: + values [ins->dreg] = LLVMBuildLShr (builder, lhs, imm, dname); break; - case OP_TRUNC_I4: - values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname); - break; - case OP_LOCALLOC_IMM: { - LLVMValueRef v; - - guint32 size = ins->inst_imm; - size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1); - - v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, ""); + default: + g_assert_not_reached (); + } + break; + } + case OP_INEG: + values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), convert (ctx, lhs, LLVMInt32Type ()), dname); + break; + case OP_LNEG: + values [ins->dreg] = LLVMBuildSub (builder, LLVMConstInt (LLVMInt64Type (), 0, FALSE), lhs, dname); + break; + case OP_FNEG: + lhs = convert (ctx, lhs, LLVMDoubleType ()); + values [ins->dreg] = LLVMBuildFSub (builder, LLVMConstReal (LLVMDoubleType (), 0.0), lhs, dname); + break; + case OP_INOT: { + guint32 v = 0xffffffff; + values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt32Type (), v, FALSE), lhs, dname); + break; + } + case OP_LNOT: { + guint64 v = 0xffffffffffffffffLL; + values [ins->dreg] = LLVMBuildXor (builder, LLVMConstInt (LLVMInt64Type (), v, FALSE), lhs, dname); + break; + } +#if defined(TARGET_X86) || defined(TARGET_AMD64) + case OP_X86_LEA: { + LLVMValueRef v1, v2; - if (ins->flags & MONO_INST_INIT) { - LLVMValueRef args [4]; + v1 = LLVMBuildMul (builder, convert (ctx, rhs, IntPtrType ()), LLVMConstInt (IntPtrType (), (1 << ins->backend.shift_amount), FALSE), ""); + v2 = LLVMBuildAdd (builder, convert (ctx, lhs, IntPtrType ()), v1, ""); + values [ins->dreg] = LLVMBuildAdd (builder, v2, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), dname); + break; + } +#endif - args [0] = v; - args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE); - args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE); - args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE); - LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memset.i32"), args, 4, ""); - } + case OP_ICONV_TO_I1: + case OP_ICONV_TO_I2: + case OP_ICONV_TO_I4: + case OP_ICONV_TO_U1: + case OP_ICONV_TO_U2: + case OP_ICONV_TO_U4: + case OP_LCONV_TO_I1: + case OP_LCONV_TO_I2: + case OP_LCONV_TO_U1: + case OP_LCONV_TO_U2: + case OP_LCONV_TO_U4: { + gboolean sign; + + sign = (ins->opcode == OP_ICONV_TO_I1) || (ins->opcode == OP_ICONV_TO_I2) || (ins->opcode == OP_ICONV_TO_I4) || (ins->opcode == OP_LCONV_TO_I1) || (ins->opcode == OP_LCONV_TO_I2); + + /* Have to do two casts since our vregs have type int */ + v = LLVMBuildTrunc (builder, lhs, op_to_llvm_type (ins->opcode), ""); + if (sign) + values [ins->dreg] = LLVMBuildSExt (builder, v, LLVMInt32Type (), dname); + else + values [ins->dreg] = LLVMBuildZExt (builder, v, LLVMInt32Type (), dname); + break; + } + case OP_ICONV_TO_I8: + values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname); + break; + case OP_ICONV_TO_U8: + values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname); + break; + case OP_FCONV_TO_I4: + values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt32Type (), dname); + break; + case OP_FCONV_TO_I1: + values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), ""); + break; + case OP_FCONV_TO_U1: + values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt8Type (), dname), LLVMInt32Type (), ""); + break; + case OP_FCONV_TO_I2: + values [ins->dreg] = LLVMBuildSExt (builder, LLVMBuildFPToSI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), ""); + break; + case OP_FCONV_TO_U2: + values [ins->dreg] = LLVMBuildZExt (builder, LLVMBuildFPToUI (builder, lhs, LLVMInt16Type (), dname), LLVMInt32Type (), ""); + break; + case OP_FCONV_TO_I8: + values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, LLVMInt64Type (), dname); + break; + case OP_FCONV_TO_I: + values [ins->dreg] = LLVMBuildFPToSI (builder, lhs, IntPtrType (), dname); + break; + case OP_ICONV_TO_R8: + case OP_LCONV_TO_R8: + values [ins->dreg] = LLVMBuildSIToFP (builder, lhs, LLVMDoubleType (), dname); + break; + case OP_LCONV_TO_R_UN: + values [ins->dreg] = LLVMBuildUIToFP (builder, lhs, LLVMDoubleType (), dname); + break; +#if SIZEOF_VOID_P == 4 + case OP_LCONV_TO_U: +#endif + case OP_LCONV_TO_I4: + values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname); + break; + case OP_ICONV_TO_R4: + case OP_LCONV_TO_R4: + v = LLVMBuildSIToFP (builder, lhs, LLVMFloatType (), ""); + values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname); + break; + case OP_FCONV_TO_R4: + v = LLVMBuildFPTrunc (builder, lhs, LLVMFloatType (), ""); + values [ins->dreg] = LLVMBuildFPExt (builder, v, LLVMDoubleType (), dname); + break; + case OP_SEXT_I4: + values [ins->dreg] = LLVMBuildSExt (builder, lhs, LLVMInt64Type (), dname); + break; + case OP_ZEXT_I4: + values [ins->dreg] = LLVMBuildZExt (builder, lhs, LLVMInt64Type (), dname); + break; + case OP_TRUNC_I4: + values [ins->dreg] = LLVMBuildTrunc (builder, lhs, LLVMInt32Type (), dname); + break; + case OP_LOCALLOC_IMM: { + LLVMValueRef v; - values [ins->dreg] = v; - break; - } - case OP_LOCALLOC: { - LLVMValueRef v, size; - - size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, lhs, LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), ""); + guint32 size = ins->inst_imm; + size = (size + (MONO_ARCH_FRAME_ALIGNMENT - 1)) & ~ (MONO_ARCH_FRAME_ALIGNMENT - 1); - v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, ""); + v = mono_llvm_build_alloca (builder, LLVMInt8Type (), LLVMConstInt (LLVMInt32Type (), size, FALSE), MONO_ARCH_FRAME_ALIGNMENT, ""); - if (ins->flags & MONO_INST_INIT) { - LLVMValueRef args [4]; + if (ins->flags & MONO_INST_INIT) { + LLVMValueRef args [5]; - args [0] = v; - args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE); - args [2] = size; - args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE); - LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memset.i32"), args, 4, ""); - } - values [ins->dreg] = v; - break; + args [0] = v; + args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE); + args [2] = LLVMConstInt (LLVMInt32Type (), size, FALSE); + args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE); + args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); + LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, ""); } - case OP_LOADI1_MEMBASE: - case OP_LOADU1_MEMBASE: - case OP_LOADI2_MEMBASE: - case OP_LOADU2_MEMBASE: - case OP_LOADI4_MEMBASE: - case OP_LOADU4_MEMBASE: - case OP_LOADI8_MEMBASE: - case OP_LOADR4_MEMBASE: - case OP_LOADR8_MEMBASE: - case OP_LOAD_MEMBASE: - case OP_LOADI8_MEM: - case OP_LOADU1_MEM: - case OP_LOADU2_MEM: - case OP_LOADI4_MEM: - case OP_LOADU4_MEM: - case OP_LOAD_MEM: { - int size = 8; - LLVMValueRef index, addr; - LLVMTypeRef t; - gboolean sext = FALSE, zext = FALSE; - gboolean is_volatile = (ins->flags & MONO_INST_FAULT); - - t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext); - - if (sext || zext) - dname = (char*)""; - - /* - * We emit volatile loads for loads which can fault, because otherwise - * LLVM will generate invalid code when encountering a load from a - * NULL address. - */ - if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) { - addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE); - } else if (ins->inst_offset == 0) { - addr = values [ins->inst_basereg]; - } else if (ins->inst_offset % size != 0) { - /* Unaligned load */ - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE); - addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, ""); - } else { - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); - addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, ""); - } + values [ins->dreg] = v; + break; + } + case OP_LOCALLOC: { + LLVMValueRef v, size; + + size = LLVMBuildAnd (builder, LLVMBuildAdd (builder, convert (ctx, lhs, LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT - 1, FALSE), ""), LLVMConstInt (LLVMInt32Type (), ~ (MONO_ARCH_FRAME_ALIGNMENT - 1), FALSE), ""); - addr = convert (ctx, addr, LLVMPointerType (t, 0)); + v = mono_llvm_build_alloca (builder, LLVMInt8Type (), size, MONO_ARCH_FRAME_ALIGNMENT, ""); - values [ins->dreg] = mono_llvm_build_load (builder, addr, dname, is_volatile); + if (ins->flags & MONO_INST_INIT) { + LLVMValueRef args [5]; - if (sext) - values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname); - else if (zext) - values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname); - else if (ins->opcode == OP_LOADR4_MEMBASE) - values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname); - break; - } - - case OP_STOREI1_MEMBASE_REG: - case OP_STOREI2_MEMBASE_REG: - case OP_STOREI4_MEMBASE_REG: - case OP_STOREI8_MEMBASE_REG: - case OP_STORER4_MEMBASE_REG: - case OP_STORER8_MEMBASE_REG: - case OP_STORE_MEMBASE_REG: { - int size = 8; - LLVMValueRef index, addr; - LLVMTypeRef t; - gboolean sext = FALSE, zext = FALSE; - - t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext); - - g_assert (ins->inst_offset % size == 0); - - if (ins->inst_offset % size != 0) { - /* Unaligned store */ - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE); - addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, ""); - } else { - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); - addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, ""); - } - LLVMBuildStore (builder, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0))); - break; + args [0] = v; + args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE); + args [2] = size; + args [3] = LLVMConstInt (LLVMInt32Type (), MONO_ARCH_FRAME_ALIGNMENT, FALSE); + args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); + LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, ""); } + values [ins->dreg] = v; + break; + } - case OP_STOREI1_MEMBASE_IMM: - case OP_STOREI2_MEMBASE_IMM: - case OP_STOREI4_MEMBASE_IMM: - case OP_STOREI8_MEMBASE_IMM: - case OP_STORE_MEMBASE_IMM: { - int size = 8; - LLVMValueRef index, addr; - LLVMTypeRef t; - gboolean sext = FALSE, zext = FALSE; - - t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext); - - if (ins->inst_offset % size != 0) { - /* Unaligned store */ - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE); - addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, ""); - } else { - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); - addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, ""); - } - LLVMBuildStore (builder, convert (ctx, LLVMConstInt (LLVMInt32Type (), ins->inst_imm, FALSE), t), addr); - break; + case OP_LOADI1_MEMBASE: + case OP_LOADU1_MEMBASE: + case OP_LOADI2_MEMBASE: + case OP_LOADU2_MEMBASE: + case OP_LOADI4_MEMBASE: + case OP_LOADU4_MEMBASE: + case OP_LOADI8_MEMBASE: + case OP_LOADR4_MEMBASE: + case OP_LOADR8_MEMBASE: + case OP_LOAD_MEMBASE: + case OP_LOADI8_MEM: + case OP_LOADU1_MEM: + case OP_LOADU2_MEM: + case OP_LOADI4_MEM: + case OP_LOADU4_MEM: + case OP_LOAD_MEM: { + int size = 8; + LLVMValueRef index, addr; + LLVMTypeRef t; + gboolean sext = FALSE, zext = FALSE; + gboolean is_volatile = (ins->flags & MONO_INST_FAULT); + + t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext); + + if (sext || zext) + dname = (char*)""; + + if ((ins->opcode == OP_LOADI8_MEM) || (ins->opcode == OP_LOAD_MEM) || (ins->opcode == OP_LOADI4_MEM) || (ins->opcode == OP_LOADU4_MEM) || (ins->opcode == OP_LOADU1_MEM) || (ins->opcode == OP_LOADU2_MEM)) { + addr = LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE); + } else if (ins->inst_offset == 0) { + addr = values [ins->inst_basereg]; + } else if (ins->inst_offset % size != 0) { + /* Unaligned load */ + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE); + addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, ""); + } else { + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); + addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (t, 0)), &index, 1, ""); } - case OP_CHECK_THIS: - mono_llvm_build_load (builder, convert (ctx, values [ins->sreg1], LLVMPointerType (IntPtrType (), 0)), "", TRUE); - break; - case OP_OUTARG_VTRETADDR: - break; - case OP_VOIDCALL: - case OP_CALL: - case OP_LCALL: - case OP_FCALL: - case OP_VCALL: - case OP_VOIDCALL_MEMBASE: - case OP_CALL_MEMBASE: - case OP_LCALL_MEMBASE: - case OP_FCALL_MEMBASE: - case OP_VCALL_MEMBASE: - case OP_VOIDCALL_REG: - case OP_CALL_REG: - case OP_LCALL_REG: - case OP_FCALL_REG: - case OP_VCALL_REG: { - MonoCallInst *call = (MonoCallInst*)ins; - MonoMethodSignature *sig = call->signature; - LLVMValueRef callee, lcall; - LLVMValueRef *args; - LLVMCallInfo *cinfo; - GSList *l; - int i, pindex; - gboolean vretaddr; - LLVMTypeRef llvm_sig; - gpointer target; - int *pindexes; - gboolean virtual, calli; - - cinfo = call->cinfo; - - vretaddr = cinfo && cinfo->ret.storage == LLVMArgVtypeRetAddr; - - llvm_sig = sig_to_llvm_sig (ctx, sig, cinfo); - CHECK_FAILURE (ctx); - - virtual = (ins->opcode == OP_VOIDCALL_MEMBASE || ins->opcode == OP_CALL_MEMBASE || ins->opcode == OP_VCALL_MEMBASE || ins->opcode == OP_LCALL_MEMBASE || ins->opcode == OP_FCALL_MEMBASE); - calli = (ins->opcode == OP_VOIDCALL_REG || ins->opcode == OP_CALL_REG || ins->opcode == OP_VCALL_REG || ins->opcode == OP_LCALL_REG || ins->opcode == OP_FCALL_REG); - - pindexes = mono_mempool_alloc0 (cfg->mempool, (sig->param_count + 2) * sizeof (guint32)); - - /* FIXME: Avoid creating duplicate methods */ - - if (ins->flags & MONO_INST_HAS_METHOD) { - if (virtual) { - callee = NULL; - } else { - if (cfg->compile_aot) { - callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_METHOD, call->method); - if (!callee) - LLVM_FAILURE (ctx, "can't encode patch"); - } else { - callee = LLVMAddFunction (module, "", llvm_sig); - - target = - mono_create_jit_trampoline_in_domain (mono_domain_get (), - call->method); - LLVMAddGlobalMapping (ee, callee, target); - } - } - } else if (calli) { - } else { - MonoJitICallInfo *info = mono_find_jit_icall_by_addr (call->fptr); - - if (info) { - /* - MonoJumpInfo ji; - - memset (&ji, 0, sizeof (ji)); - ji.type = MONO_PATCH_INFO_JIT_ICALL_ADDR; - ji.data.target = info->name; - - target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, &ji, FALSE); - */ - if (cfg->compile_aot) { - callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_INTERNAL_METHOD, (char*)info->name); - if (!callee) - LLVM_FAILURE (ctx, "can't encode patch"); - } else { - callee = LLVMAddFunction (module, "", llvm_sig); - target = (gpointer)mono_icall_get_wrapper (info); - LLVMAddGlobalMapping (ee, callee, target); - } - } else { - if (cfg->compile_aot) { - callee = NULL; - if (cfg->abs_patches) { - MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr); - if (abs_ji) { - callee = get_plt_entry (ctx, llvm_sig, abs_ji->type, abs_ji->data.target); - if (!callee) - LLVM_FAILURE (ctx, "can't encode patch"); - } - } - if (!callee) - LLVM_FAILURE (ctx, "aot"); - } else { - callee = LLVMAddFunction (module, "", llvm_sig); - target = NULL; - if (cfg->abs_patches) { - MonoJumpInfo *abs_ji = g_hash_table_lookup (cfg->abs_patches, call->fptr); - if (abs_ji) { - /* - * The monitor entry/exit trampolines might have - * their own calling convention on some platforms. - */ -#ifndef TARGET_AMD64 - if (abs_ji->type == MONO_PATCH_INFO_MONITOR_ENTER || abs_ji->type == MONO_PATCH_INFO_MONITOR_EXIT) - LLVM_FAILURE (ctx, "monitor enter/exit"); -#endif - target = mono_resolve_patch_target (cfg->method, cfg->domain, NULL, abs_ji, FALSE); - LLVMAddGlobalMapping (ee, callee, target); - } - } - if (!target) - LLVMAddGlobalMapping (ee, callee, (gpointer)call->fptr); - } - } - } - - if (virtual) { - int size = sizeof (gpointer); - LLVMValueRef index; - - g_assert (ins->inst_offset % size == 0); - index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); - - // FIXME: mono_arch_get_vcall_slot () can't decode the code - // generated by LLVM - //LLVM_FAILURE (ctx, "virtual call"); - - if (call->method && call->method->klass->flags & TYPE_ATTRIBUTE_INTERFACE) { -#ifdef MONO_ARCH_HAVE_LLVM_IMT_TRAMPOLINE - if (cfg->compile_aot) { - MonoJumpInfoImtTramp *imt_tramp = g_new0 (MonoJumpInfoImtTramp, 1); - imt_tramp->method = call->method; - imt_tramp->vt_offset = call->inst.inst_offset; - - callee = get_plt_entry (ctx, llvm_sig, MONO_PATCH_INFO_LLVM_IMT_TRAMPOLINE, imt_tramp); - } else { - callee = LLVMAddFunction (module, "", llvm_sig); - target = mono_create_llvm_imt_trampoline (cfg->domain, call->method, call->inst.inst_offset); - LLVMAddGlobalMapping (ee, callee, target); - } -#else - /* No support for passing the IMT argument */ - LLVM_FAILURE (ctx, "imt"); -#endif - } else { - callee = convert (ctx, LLVMBuildLoad (builder, LLVMBuildGEP (builder, convert (ctx, values [ins->inst_basereg], LLVMPointerType (LLVMPointerType (IntPtrType (), 0), 0)), &index, 1, ""), ""), LLVMPointerType (llvm_sig, 0)); - } - } else if (calli) { - callee = convert (ctx, values [ins->sreg1], LLVMPointerType (llvm_sig, 0)); - } else { - if (ins->flags & MONO_INST_HAS_METHOD) { - } - } - - /* - * Collect and convert arguments - */ - - args = alloca (sizeof (LLVMValueRef) * ((sig->param_count * 2) + sig->hasthis + vretaddr)); - l = call->out_ireg_args; - pindex = 0; - if (vretaddr) { - if (!addresses [call->inst.dreg]) - addresses [call->inst.dreg] = build_alloca (ctx, sig->ret); - args [pindex ++] = LLVMBuildPtrToInt (builder, addresses [call->inst.dreg], IntPtrType (), ""); - } - - for (i = 0; i < sig->param_count + sig->hasthis; ++i) { - guint32 regpair; - int reg; - LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL; - - regpair = (guint32)(gssize)(l->data); - reg = regpair & 0xffffff; - args [pindex] = values [reg]; - if (ainfo->storage == LLVMArgVtypeInReg) { - int j; - LLVMValueRef regs [2]; - guint32 nregs; - - g_assert (ainfo); - - g_assert (addresses [reg]); - - emit_vtype_to_reg (ctx, builder, sig->params [i - sig->hasthis], addresses [reg], ainfo, regs, &nregs); - for (j = 0; j < nregs; ++j) - args [pindex ++] = regs [j]; - - // FIXME: alignment - // FIXME: Get rid of the VMOVE - } else if (ainfo->storage == LLVMArgVtypeByVal) { - g_assert (addresses [reg]); - args [pindex] = addresses [reg]; - pindexes [i] = pindex; - pindex ++; - } else { - g_assert (args [pindex]); - if (i == 0 && sig->hasthis) - args [pindex] = convert (ctx, args [pindex], IntPtrType ()); - else - args [pindex] = convert (ctx, args [pindex], type_to_llvm_arg_type (ctx, sig->params [i - sig->hasthis])); - pindex ++; - } - - l = l->next; - } - - // FIXME: Align call sites - - /* - * Emit the call - */ - - lcall = emit_call (ctx, bb, &builder, callee, args, pindex); - - /* Add byval attributes if needed */ - for (i = 0; i < sig->param_count + sig->hasthis; ++i) { - LLVMArgInfo *ainfo = call->cinfo ? &call->cinfo->args [i] : NULL; + addr = convert (ctx, addr, LLVMPointerType (t, 0)); - if (ainfo && ainfo->storage == LLVMArgVtypeByVal) { - LLVMAddInstrAttribute (lcall, pindexes [i] + 1, LLVMByValAttribute); - } - } + values [ins->dreg] = emit_load (ctx, bb, &builder, size, addr, dname, is_volatile); + if (!is_volatile && (ins->flags & MONO_INST_CONSTANT_LOAD)) { /* - * Convert the result + * These will signal LLVM that these loads do not alias any stores, and + * they can't fail, allowing them to be hoisted out of loops. */ - if (cinfo && cinfo->ret.storage == LLVMArgVtypeInReg) { - LLVMValueRef regs [2]; - - if (!addresses [ins->dreg]) - addresses [ins->dreg] = build_alloca (ctx, sig->ret); + set_metadata_flag (values [ins->dreg], "mono.noalias"); + set_metadata_flag (values [ins->dreg], "mono.nofail.load"); + } - regs [0] = LLVMBuildExtractValue (builder, lcall, 0, ""); - if (cinfo->ret.pair_storage [1] != LLVMArgNone) - regs [1] = LLVMBuildExtractValue (builder, lcall, 1, ""); - - emit_reg_to_vtype (ctx, builder, sig->ret, addresses [ins->dreg], &cinfo->ret, regs); - } else if (sig->ret->type != MONO_TYPE_VOID && !vretaddr) { - /* If the method returns an unsigned value, need to zext it */ + if (sext) + values [ins->dreg] = LLVMBuildSExt (builder, values [ins->dreg], LLVMInt32Type (), dname); + else if (zext) + values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), dname); + else if (ins->opcode == OP_LOADR4_MEMBASE) + values [ins->dreg] = LLVMBuildFPExt (builder, values [ins->dreg], LLVMDoubleType (), dname); + break; + } + + case OP_STOREI1_MEMBASE_REG: + case OP_STOREI2_MEMBASE_REG: + case OP_STOREI4_MEMBASE_REG: + case OP_STOREI8_MEMBASE_REG: + case OP_STORER4_MEMBASE_REG: + case OP_STORER8_MEMBASE_REG: + case OP_STORE_MEMBASE_REG: { + int size = 8; + LLVMValueRef index, addr; + LLVMTypeRef t; + gboolean sext = FALSE, zext = FALSE; + gboolean is_volatile = (ins->flags & MONO_INST_FAULT); + + if (!values [ins->inst_destbasereg]) + LLVM_FAILURE (ctx, "inst_destbasereg"); + + t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext); + + if (ins->inst_offset % size != 0) { + /* Unaligned store */ + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE); + addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, ""); + } else { + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); + addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, ""); + } + emit_store (ctx, bb, &builder, size, convert (ctx, values [ins->sreg1], t), convert (ctx, addr, LLVMPointerType (t, 0)), is_volatile); + break; + } - values [ins->dreg] = convert_full (ctx, lcall, llvm_type_to_stack_type (type_to_llvm_type (ctx, sig->ret)), type_is_unsigned (ctx, sig->ret)); - } - break; + case OP_STOREI1_MEMBASE_IMM: + case OP_STOREI2_MEMBASE_IMM: + case OP_STOREI4_MEMBASE_IMM: + case OP_STOREI8_MEMBASE_IMM: + case OP_STORE_MEMBASE_IMM: { + int size = 8; + LLVMValueRef index, addr; + LLVMTypeRef t; + gboolean sext = FALSE, zext = FALSE; + gboolean is_volatile = (ins->flags & MONO_INST_FAULT); + + t = load_store_to_llvm_type (ins->opcode, &size, &sext, &zext); + + if (ins->inst_offset % size != 0) { + /* Unaligned store */ + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset, FALSE); + addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (LLVMInt8Type (), 0)), &index, 1, ""); + } else { + index = LLVMConstInt (LLVMInt32Type (), ins->inst_offset / size, FALSE); + addr = LLVMBuildGEP (builder, convert (ctx, values [ins->inst_destbasereg], LLVMPointerType (t, 0)), &index, 1, ""); } - case OP_AOTCONST: { - guint32 got_offset; - LLVMValueRef indexes [2]; - MonoJumpInfo *ji; - LLVMValueRef got_entry_addr; + emit_store (ctx, bb, &builder, size, convert (ctx, LLVMConstInt (IntPtrType (), ins->inst_imm, FALSE), t), addr, is_volatile); + break; + } - /* - * FIXME: Can't allocate from the cfg mempool since that is freed if - * the LLVM compile fails. - */ - ji = g_new0 (MonoJumpInfo, 1); - ji->type = (MonoJumpInfoType)ins->inst_i1; - ji->data.target = ins->inst_p0; + case OP_CHECK_THIS: + emit_load (ctx, bb, &builder, sizeof (gpointer), convert (ctx, values [ins->sreg1], LLVMPointerType (IntPtrType (), 0)), "", TRUE); + break; + case OP_OUTARG_VTRETADDR: + break; + case OP_VOIDCALL: + case OP_CALL: + case OP_LCALL: + case OP_FCALL: + case OP_VCALL: + case OP_VOIDCALL_MEMBASE: + case OP_CALL_MEMBASE: + case OP_LCALL_MEMBASE: + case OP_FCALL_MEMBASE: + case OP_VCALL_MEMBASE: + case OP_VOIDCALL_REG: + case OP_CALL_REG: + case OP_LCALL_REG: + case OP_FCALL_REG: + case OP_VCALL_REG: { + process_call (ctx, bb, &builder, ins); + CHECK_FAILURE (ctx); + break; + } + case OP_AOTCONST: { + guint32 got_offset; + LLVMValueRef indexes [2]; + MonoJumpInfo *ji; + LLVMValueRef got_entry_addr; - ji = mono_aot_patch_info_dup (ji); + /* + * FIXME: Can't allocate from the cfg mempool since that is freed if + * the LLVM compile fails. + */ + ji = g_new0 (MonoJumpInfo, 1); + ji->type = (MonoJumpInfoType)ins->inst_i1; + ji->data.target = ins->inst_p0; + + ji = mono_aot_patch_info_dup (ji); - ji->next = cfg->patch_info; - cfg->patch_info = ji; + ji->next = cfg->patch_info; + cfg->patch_info = ji; - //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0); - got_offset = mono_aot_get_got_offset (cfg->patch_info); + //mono_add_patch_info (cfg, 0, (MonoJumpInfoType)ins->inst_i1, ins->inst_p0); + got_offset = mono_aot_get_got_offset (cfg->patch_info); - indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE); - got_entry_addr = LLVMBuildGEP (builder, ctx->lmodule->got_var, indexes, 2, ""); + indexes [0] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + indexes [1] = LLVMConstInt (LLVMInt32Type (), (gssize)got_offset, FALSE); + got_entry_addr = LLVMBuildGEP (builder, ctx->lmodule->got_var, indexes, 2, ""); - // FIXME: This doesn't work right now, because it must be - // paired with an invariant.end, and even then, its only in effect - // inside its basic block + // FIXME: This doesn't work right now, because it must be + // paired with an invariant.end, and even then, its only in effect + // inside its basic block #if 0 - { - LLVMValueRef args [3]; - LLVMValueRef ptr, val; + { + LLVMValueRef args [3]; + LLVMValueRef ptr, val; - ptr = LLVMBuildBitCast (builder, got_entry_addr, LLVMPointerType (LLVMInt8Type (), 0), "ptr"); + ptr = LLVMBuildBitCast (builder, got_entry_addr, LLVMPointerType (LLVMInt8Type (), 0), "ptr"); - args [0] = LLVMConstInt (LLVMInt64Type (), sizeof (gpointer), FALSE); - args [1] = ptr; - val = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.invariant.start"), args, 2, ""); - } + args [0] = LLVMConstInt (LLVMInt64Type (), sizeof (gpointer), FALSE); + args [1] = ptr; + val = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.invariant.start"), args, 2, ""); + } #endif - values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, dname); - break; - } - case OP_NOT_REACHED: - LLVMBuildUnreachable (builder); - has_terminator = TRUE; - g_assert (bb->block_num < cfg->max_block_num); - unreachable [bb->block_num] = TRUE; - /* Might have instructions after this */ - while (ins->next) { - MonoInst *next = ins->next; - MONO_DELETE_INS (bb, next); - } - break; - case OP_LDADDR: { - MonoInst *var = ins->inst_p0; + values [ins->dreg] = LLVMBuildLoad (builder, got_entry_addr, dname); + break; + } + case OP_NOT_REACHED: + LLVMBuildUnreachable (builder); + has_terminator = TRUE; + g_assert (bb->block_num < cfg->max_block_num); + ctx->unreachable [bb->block_num] = TRUE; + /* Might have instructions after this */ + while (ins->next) { + MonoInst *next = ins->next; + /* + * FIXME: If later code uses the regs defined by these instructions, + * compilation will fail. + */ + MONO_DELETE_INS (bb, next); + } + break; + case OP_LDADDR: { + MonoInst *var = ins->inst_p0; - values [ins->dreg] = addresses [var->dreg]; - break; - } - case OP_SIN: { - LLVMValueRef args [1]; + values [ins->dreg] = addresses [var->dreg]; + break; + } + case OP_SIN: { + LLVMValueRef args [1]; - args [0] = convert (ctx, lhs, LLVMDoubleType ()); - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sin.f64"), args, 1, dname); - break; - } - case OP_COS: { - LLVMValueRef args [1]; + args [0] = convert (ctx, lhs, LLVMDoubleType ()); + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sin.f64"), args, 1, dname); + break; + } + case OP_COS: { + LLVMValueRef args [1]; - args [0] = convert (ctx, lhs, LLVMDoubleType ()); - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.cos.f64"), args, 1, dname); - break; - } - /* test_0_sqrt_nan fails with LLVM */ - /* - case OP_SQRT: { - LLVMValueRef args [1]; + args [0] = convert (ctx, lhs, LLVMDoubleType ()); + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.cos.f64"), args, 1, dname); + break; + } + case OP_SQRT: { + LLVMValueRef args [1]; - args [0] = lhs; - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sqrt.f64"), args, 1, dname); - break; - } - */ + /* + * LLVM optimizes sqrt(nan) into undefined in + * lib/Analysis/ConstantFolding.cpp + * Also, sqrt(NegativeInfinity) is optimized into 0. + */ + LLVM_FAILURE (ctx, "sqrt"); + args [0] = lhs; + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.sqrt.f64"), args, 1, dname); + break; + } + case OP_ABS: { + LLVMValueRef args [1]; + + args [0] = lhs; + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "fabs"), args, 1, dname); + break; + } - case OP_ABS: { - LLVMValueRef args [1]; + case OP_IMIN: + case OP_LMIN: + case OP_IMAX: + case OP_LMAX: + case OP_IMIN_UN: + case OP_LMIN_UN: + case OP_IMAX_UN: + case OP_LMAX_UN: { + LLVMValueRef v; - args [0] = lhs; - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "fabs"), args, 1, dname); - break; - } + lhs = convert (ctx, lhs, regtype_to_llvm_type (spec [MONO_INST_DEST])); + rhs = convert (ctx, rhs, regtype_to_llvm_type (spec [MONO_INST_DEST])); + switch (ins->opcode) { case OP_IMIN: - case OP_LMIN: { - LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname); + case OP_LMIN: + v = LLVMBuildICmp (builder, LLVMIntSLE, lhs, rhs, ""); break; - } case OP_IMAX: - case OP_LMAX: { - LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname); + case OP_LMAX: + v = LLVMBuildICmp (builder, LLVMIntSGE, lhs, rhs, ""); break; - } case OP_IMIN_UN: - case OP_LMIN_UN: { - LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname); + case OP_LMIN_UN: + v = LLVMBuildICmp (builder, LLVMIntULE, lhs, rhs, ""); break; - } case OP_IMAX_UN: - case OP_LMAX_UN: { - LLVMValueRef v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname); + case OP_LMAX_UN: + v = LLVMBuildICmp (builder, LLVMIntUGE, lhs, rhs, ""); + break; + default: + g_assert_not_reached (); break; } - case OP_ATOMIC_EXCHANGE_I4: { - LLVMValueRef args [2]; + values [ins->dreg] = LLVMBuildSelect (builder, v, lhs, rhs, dname); + break; + } + case OP_ATOMIC_EXCHANGE_I4: { + LLVMValueRef args [2]; - g_assert (ins->inst_offset == 0); + g_assert (ins->inst_offset == 0); - args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt32Type (), 0)); - args [1] = rhs; - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.swap.i32.p0i32"), args, 2, dname); - break; - } - case OP_ATOMIC_EXCHANGE_I8: { - LLVMValueRef args [2]; + args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt32Type (), 0)); + args [1] = rhs; + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.swap.i32.p0i32"), args, 2, dname); + break; + } + case OP_ATOMIC_EXCHANGE_I8: { + LLVMValueRef args [2]; - g_assert (ins->inst_offset == 0); + g_assert (ins->inst_offset == 0); - args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt64Type (), 0)); - args [1] = rhs; - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.swap.i64.p0i64"), args, 2, dname); - break; - } - case OP_ATOMIC_ADD_NEW_I4: { - LLVMValueRef args [2]; + args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt64Type (), 0)); + args [1] = convert (ctx, rhs, LLVMInt64Type ()); + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.swap.i64.p0i64"), args, 2, dname); + break; + } + case OP_ATOMIC_ADD_NEW_I4: { + LLVMValueRef args [2]; - g_assert (ins->inst_offset == 0); + g_assert (ins->inst_offset == 0); - args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt32Type (), 0)); - args [1] = rhs; - values [ins->dreg] = LLVMBuildAdd (builder, LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.load.add.i32.p0i32"), args, 2, ""), args [1], dname); - break; - } - case OP_ATOMIC_ADD_NEW_I8: { - LLVMValueRef args [2]; + args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt32Type (), 0)); + args [1] = rhs; + values [ins->dreg] = LLVMBuildAdd (builder, LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.load.add.i32.p0i32"), args, 2, ""), args [1], dname); + break; + } + case OP_ATOMIC_ADD_NEW_I8: { + LLVMValueRef args [2]; - g_assert (ins->inst_offset == 0); + g_assert (ins->inst_offset == 0); - args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt64Type (), 0)); - args [1] = convert (ctx, rhs, LLVMInt64Type ()); - values [ins->dreg] = LLVMBuildAdd (builder, LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.load.add.i64.p0i64"), args, 2, ""), args [1], dname); - break; - } - case OP_ATOMIC_CAS_I4: - case OP_ATOMIC_CAS_I8: { - LLVMValueRef args [3]; - LLVMTypeRef t; - const char *intrins; + args [0] = convert (ctx, lhs, LLVMPointerType (LLVMInt64Type (), 0)); + args [1] = convert (ctx, rhs, LLVMInt64Type ()); + values [ins->dreg] = LLVMBuildAdd (builder, LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.atomic.load.add.i64.p0i64"), args, 2, ""), args [1], dname); + break; + } + case OP_ATOMIC_CAS_I4: + case OP_ATOMIC_CAS_I8: { + LLVMValueRef args [3]; + LLVMTypeRef t; + const char *intrins; - if (ins->opcode == OP_ATOMIC_CAS_I4) { - t = LLVMInt32Type (); - intrins = "llvm.atomic.cmp.swap.i32.p0i32"; - } else { - t = LLVMInt64Type (); - intrins = "llvm.atomic.cmp.swap.i64.p0i64"; - } - - args [0] = convert (ctx, lhs, LLVMPointerType (t, 0)); - /* comparand */ - args [1] = convert (ctx, values [ins->sreg3], t); - /* new value */ - args [2] = convert (ctx, values [ins->sreg2], t); - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, intrins), args, 3, dname); - break; + if (ins->opcode == OP_ATOMIC_CAS_I4) { + t = LLVMInt32Type (); + intrins = "llvm.atomic.cmp.swap.i32.p0i32"; + } else { + t = LLVMInt64Type (); + intrins = "llvm.atomic.cmp.swap.i64.p0i64"; } - case OP_MEMORY_BARRIER: { - LLVMValueRef args [5]; - for (i = 0; i < 5; ++i) - args [i] = LLVMConstInt (LLVMInt1Type (), TRUE, TRUE); + args [0] = convert (ctx, lhs, LLVMPointerType (t, 0)); + /* comparand */ + args [1] = convert (ctx, values [ins->sreg3], t); + /* new value */ + args [2] = convert (ctx, values [ins->sreg2], t); + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, intrins), args, 3, dname); + break; + } + case OP_MEMORY_BARRIER: { + LLVMValueRef args [5]; - LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memory.barrier"), args, 5, ""); - break; - } - case OP_RELAXED_NOP: { +#ifdef TARGET_ARM + /* Not yet supported by llc on arm */ + LLVM_FAILURE (ctx, "memory-barrier+arm"); +#endif + + for (i = 0; i < 5; ++i) + args [i] = LLVMConstInt (LLVMInt1Type (), TRUE, TRUE); + + LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memory.barrier"), args, 5, ""); + break; + } + case OP_RELAXED_NOP: { #if defined(TARGET_AMD64) || defined(TARGET_X86) - /* No way to get LLVM to emit this */ - LLVM_FAILURE (ctx, "relaxed_nop"); + /* No way to get LLVM to emit this */ + LLVM_FAILURE (ctx, "relaxed_nop"); #else - break; + break; #endif - } - case OP_TLS_GET: { + } + case OP_TLS_GET: { #if defined(TARGET_AMD64) || defined(TARGET_X86) #ifdef TARGET_AMD64 - // 257 == FS segment register - LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257); + // 257 == FS segment register + LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 257); #else - // 256 == GS segment register - LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256); + // 256 == GS segment register + LLVMTypeRef ptrtype = LLVMPointerType (IntPtrType (), 256); #endif - // FIXME: XEN - values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), ""); + // FIXME: XEN + values [ins->dreg] = LLVMBuildLoad (builder, LLVMBuildIntToPtr (builder, LLVMConstInt (IntPtrType (), ins->inst_offset, TRUE), ptrtype, ""), ""); #else - LLVM_FAILURE (ctx, "opcode tls-get"); + LLVM_FAILURE (ctx, "opcode tls-get"); #endif - break; - } + break; + } /* * Overflow opcodes. */ - case OP_IADD_OVF: - case OP_IADD_OVF_UN: - case OP_ISUB_OVF: - case OP_ISUB_OVF_UN: - case OP_IMUL_OVF: - case OP_IMUL_OVF_UN: + case OP_IADD_OVF: + case OP_IADD_OVF_UN: + case OP_ISUB_OVF: + case OP_ISUB_OVF_UN: + case OP_IMUL_OVF: + case OP_IMUL_OVF_UN: #if SIZEOF_VOID_P == 8 - case OP_LADD_OVF: - case OP_LADD_OVF_UN: - case OP_LSUB_OVF: - case OP_LSUB_OVF_UN: - case OP_LMUL_OVF: - case OP_LMUL_OVF_UN: + case OP_LADD_OVF: + case OP_LADD_OVF_UN: + case OP_LSUB_OVF: + case OP_LSUB_OVF_UN: + case OP_LMUL_OVF: + case OP_LMUL_OVF_UN: #endif { LLVMValueRef args [2], val, ovf, func; - emit_cond_throw_pos (ctx); - args [0] = convert (ctx, lhs, op_to_llvm_type (ins->opcode)); args [1] = convert (ctx, rhs, op_to_llvm_type (ins->opcode)); func = LLVMGetNamedFunction (module, ovf_op_to_intrins (ins->opcode)); @@ -3006,6 +3210,7 @@ mono_llvm_emit_method (MonoCompile *cfg) values [ins->dreg] = LLVMBuildExtractValue (builder, val, 0, dname); ovf = LLVMBuildExtractValue (builder, val, 1, ""); emit_cond_system_exception (ctx, bb, "OverflowException", ovf); + CHECK_FAILURE (ctx); builder = ctx->builder; break; } @@ -3017,401 +3222,808 @@ mono_llvm_emit_method (MonoCompile *cfg) * so we always have an entry in cfg->varinfo for them. * FIXME: Is this needed ? */ - case OP_VZERO: { - MonoClass *klass = ins->klass; - LLVMValueRef args [4]; + case OP_VZERO: { + MonoClass *klass = ins->klass; + LLVMValueRef args [5]; - if (!klass) { - // FIXME: - LLVM_FAILURE (ctx, "!klass"); - break; - } + if (!klass) { + // FIXME: + LLVM_FAILURE (ctx, "!klass"); + break; + } - if (!addresses [ins->dreg]) - addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg); - args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), ""); - args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE); - args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE); - // FIXME: Alignment - args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memset.i32"), args, 4, ""); + if (!addresses [ins->dreg]) + addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg); + args [0] = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), ""); + args [1] = LLVMConstInt (LLVMInt8Type (), 0, FALSE); + args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE); + // FIXME: Alignment + args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); + LLVMBuildCall (builder, LLVMGetNamedFunction (module, memset_func_name), args, memset_param_count, ""); + break; + } + + case OP_STOREV_MEMBASE: + case OP_LOADV_MEMBASE: + case OP_VMOVE: { + MonoClass *klass = ins->klass; + LLVMValueRef src, dst, args [5]; + gboolean done = FALSE; + + if (!klass) { + // FIXME: + LLVM_FAILURE (ctx, "!klass"); break; } + switch (ins->opcode) { case OP_STOREV_MEMBASE: - case OP_LOADV_MEMBASE: - case OP_VMOVE: { - MonoClass *klass = ins->klass; - LLVMValueRef src, dst, args [4]; - gboolean done = FALSE; - - if (!klass) { - // FIXME: - LLVM_FAILURE (ctx, "!klass"); + if (cfg->gen_write_barriers && klass->has_references && ins->inst_destbasereg != cfg->frame_reg) { + /* FIXME: Emit write barriers like in mini_emit_stobj () */ + LLVM_FAILURE (ctx, "storev_membase + write barriers"); break; } - - switch (ins->opcode) { - case OP_STOREV_MEMBASE: - if (!addresses [ins->sreg1]) { - /* SIMD */ - g_assert (values [ins->sreg1]); - dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0)); - LLVMBuildStore (builder, values [ins->sreg1], dst); - done = TRUE; - } else { - src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), ""); - dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0)); - } - break; - case OP_LOADV_MEMBASE: - if (!addresses [ins->dreg]) - addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg); - src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0)); - dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), ""); - break; - case OP_VMOVE: - if (!addresses [ins->sreg1]) - addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg); - if (!addresses [ins->dreg]) - addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg); + if (!addresses [ins->sreg1]) { + /* SIMD */ + g_assert (values [ins->sreg1]); + dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (type_to_llvm_type (ctx, &klass->byval_arg), 0)); + LLVMBuildStore (builder, values [ins->sreg1], dst); + done = TRUE; + } else { src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), ""); - dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), ""); - break; - default: - g_assert_not_reached (); + dst = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_destbasereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0)); } - - if (done) - break; - - args [0] = dst; - args [1] = src; - args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE); - args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - // FIXME: Alignment - args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - LLVMBuildCall (builder, LLVMGetNamedFunction (module, "llvm.memcpy.i32"), args, 4, ""); break; + case OP_LOADV_MEMBASE: + if (!addresses [ins->dreg]) + addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg); + src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (LLVMInt8Type (), 0)); + dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), ""); + break; + case OP_VMOVE: + if (!addresses [ins->sreg1]) + addresses [ins->sreg1] = build_alloca (ctx, &klass->byval_arg); + if (!addresses [ins->dreg]) + addresses [ins->dreg] = build_alloca (ctx, &klass->byval_arg); + src = LLVMBuildBitCast (builder, addresses [ins->sreg1], LLVMPointerType (LLVMInt8Type (), 0), ""); + dst = LLVMBuildBitCast (builder, addresses [ins->dreg], LLVMPointerType (LLVMInt8Type (), 0), ""); + break; + default: + g_assert_not_reached (); } - case OP_LLVM_OUTARG_VT: - if (!addresses [ins->sreg1]) { - addresses [ins->sreg1] = build_alloca (ctx, &ins->klass->byval_arg); - g_assert (values [ins->sreg1]); - LLVMBuildStore (builder, values [ins->sreg1], addresses [ins->sreg1]); - } - addresses [ins->dreg] = addresses [ins->sreg1]; + CHECK_FAILURE (ctx); + + if (done) break; + args [0] = dst; + args [1] = src; + args [2] = LLVMConstInt (LLVMInt32Type (), mono_class_value_size (klass, NULL), FALSE); + args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + // FIXME: Alignment + args [3] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + args [4] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); + LLVMBuildCall (builder, LLVMGetNamedFunction (module, memcpy_func_name), args, memcpy_param_count, ""); + break; + } + case OP_LLVM_OUTARG_VT: + if (!addresses [ins->sreg1]) { + addresses [ins->sreg1] = build_alloca (ctx, &ins->klass->byval_arg); + g_assert (values [ins->sreg1]); + LLVMBuildStore (builder, values [ins->sreg1], addresses [ins->sreg1]); + } + addresses [ins->dreg] = addresses [ins->sreg1]; + break; + /* * SIMD */ #if defined(TARGET_X86) || defined(TARGET_AMD64) - case OP_XZERO: { - values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg)); - break; - } - case OP_LOADX_MEMBASE: { - LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg); - LLVMValueRef src; + case OP_XZERO: { + values [ins->dreg] = LLVMConstNull (type_to_llvm_type (ctx, &ins->klass->byval_arg)); + break; + } + case OP_LOADX_MEMBASE: { + LLVMTypeRef t = type_to_llvm_type (ctx, &ins->klass->byval_arg); + LLVMValueRef src; - src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0)); - values [ins->dreg] = LLVMBuildLoad (builder, src, ""); - break; - } - case OP_ADDPD: - case OP_ADDPS: - case OP_PADDB: - case OP_PADDW: - case OP_PADDD: - case OP_PADDQ: - values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, ""); - break; - case OP_SUBPD: - case OP_SUBPS: - case OP_PSUBB: - case OP_PSUBW: - case OP_PSUBD: - case OP_PSUBQ: - values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, ""); - break; - case OP_MULPD: - case OP_MULPS: - values [ins->dreg] = LLVMBuildMul (builder, lhs, rhs, ""); - break; - case OP_DIVPD: - case OP_DIVPS: - values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, ""); - break; - case OP_PAND: - values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, ""); - break; - case OP_POR: - values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, ""); - break; - case OP_PXOR: - values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, ""); - break; + src = convert (ctx, LLVMBuildAdd (builder, convert (ctx, values [ins->inst_basereg], IntPtrType ()), LLVMConstInt (IntPtrType (), ins->inst_offset, FALSE), ""), LLVMPointerType (t, 0)); + values [ins->dreg] = mono_llvm_build_aligned_load (builder, src, "", FALSE, 1); + break; + } + case OP_PADDB: + case OP_PADDW: + case OP_PADDD: + case OP_PADDQ: + values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, ""); + break; + case OP_ADDPD: + case OP_ADDPS: + values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, ""); + break; + case OP_PSUBB: + case OP_PSUBW: + case OP_PSUBD: + case OP_PSUBQ: + values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, ""); + break; + case OP_SUBPD: + case OP_SUBPS: + values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, ""); + break; + case OP_MULPD: + case OP_MULPS: + values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, ""); + break; + case OP_DIVPD: + case OP_DIVPS: + values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, ""); + break; + case OP_PAND: + values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, ""); + break; + case OP_POR: + values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, ""); + break; + case OP_PXOR: + values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, ""); + break; + case OP_ANDPS: + case OP_ANDNPS: + case OP_ORPS: + case OP_XORPS: + case OP_ANDPD: + case OP_ANDNPD: + case OP_ORPD: + case OP_XORPD: { + LLVMTypeRef t, rt; + LLVMValueRef v; + + switch (ins->opcode) { case OP_ANDPS: case OP_ANDNPS: case OP_ORPS: case OP_XORPS: + t = LLVMVectorType (LLVMInt32Type (), 4); + rt = LLVMVectorType (LLVMFloatType (), 4); + break; case OP_ANDPD: case OP_ANDNPD: case OP_ORPD: - case OP_XORPD: { - LLVMTypeRef t, rt; - LLVMValueRef v; - - switch (ins->opcode) { - case OP_ANDPS: - case OP_ANDNPS: - case OP_ORPS: - case OP_XORPS: - t = LLVMVectorType (LLVMInt32Type (), 4); - rt = LLVMVectorType (LLVMFloatType (), 4); - break; - case OP_ANDPD: - case OP_ANDNPD: - case OP_ORPD: - case OP_XORPD: - t = LLVMVectorType (LLVMInt64Type (), 2); - rt = LLVMVectorType (LLVMDoubleType (), 2); - break; - default: - t = LLVMInt32Type (); - rt = LLVMInt32Type (); - g_assert_not_reached (); - } - - lhs = LLVMBuildBitCast (builder, lhs, t, ""); - rhs = LLVMBuildBitCast (builder, rhs, t, ""); - switch (ins->opcode) { - case OP_ANDPS: - case OP_ANDPD: - v = LLVMBuildAnd (builder, lhs, rhs, ""); - break; - case OP_ORPS: - case OP_ORPD: - v = LLVMBuildOr (builder, lhs, rhs, ""); - break; - case OP_XORPS: - case OP_XORPD: - v = LLVMBuildXor (builder, lhs, rhs, ""); - break; - case OP_ANDNPS: - case OP_ANDNPD: - v = LLVMBuildAnd (builder, lhs, LLVMBuildNot (builder, rhs, ""), ""); - break; - } - values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, ""); + case OP_XORPD: + t = LLVMVectorType (LLVMInt64Type (), 2); + rt = LLVMVectorType (LLVMDoubleType (), 2); break; + default: + t = LLVMInt32Type (); + rt = LLVMInt32Type (); + g_assert_not_reached (); } - case OP_MINPD: - case OP_MINPS: - case OP_MAXPD: - case OP_MAXPS: - case OP_PMIND_UN: - case OP_PMINW_UN: - case OP_PMINB_UN: - case OP_PMAXD_UN: - case OP_PMAXW_UN: - case OP_PMAXB_UN: { - LLVMValueRef args [2]; - - args [0] = lhs; - args [1] = rhs; - - values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname); + + lhs = LLVMBuildBitCast (builder, lhs, t, ""); + rhs = LLVMBuildBitCast (builder, rhs, t, ""); + switch (ins->opcode) { + case OP_ANDPS: + case OP_ANDPD: + v = LLVMBuildAnd (builder, lhs, rhs, ""); + break; + case OP_ORPS: + case OP_ORPD: + v = LLVMBuildOr (builder, lhs, rhs, ""); + break; + case OP_XORPS: + case OP_XORPD: + v = LLVMBuildXor (builder, lhs, rhs, ""); + break; + case OP_ANDNPS: + case OP_ANDNPD: + v = LLVMBuildAnd (builder, rhs, LLVMBuildNot (builder, lhs, ""), ""); break; } + values [ins->dreg] = LLVMBuildBitCast (builder, v, rt, ""); + break; + } + case OP_MINPD: + case OP_MINPS: + case OP_MAXPD: + case OP_MAXPS: + case OP_PMIND_UN: + case OP_PMINW_UN: + case OP_PMINB_UN: + case OP_PMINW: + case OP_PMAXD_UN: + case OP_PMAXW_UN: + case OP_PMAXB_UN: + case OP_PCMPEQB: + case OP_PCMPEQW: + case OP_PCMPEQD: + case OP_PCMPEQQ: + case OP_PCMPGTB: { + LLVMValueRef args [2]; + + args [0] = lhs; + args [1] = rhs; + + values [ins->dreg] = LLVMBuildCall (builder, LLVMGetNamedFunction (module, simd_op_to_intrins (ins->opcode)), args, 2, dname); + break; + } + case OP_EXTRACT_R8: + case OP_EXTRACT_I8: + case OP_EXTRACT_I4: + case OP_EXTRACT_I2: + case OP_EXTRACT_U2: + case OP_EXTRACT_I1: + case OP_EXTRACT_U1: { + LLVMTypeRef t; + gboolean zext = FALSE; + + t = simd_op_to_llvm_type (ins->opcode); + + switch (ins->opcode) { case OP_EXTRACT_R8: case OP_EXTRACT_I8: case OP_EXTRACT_I4: case OP_EXTRACT_I2: - case OP_EXTRACT_U2: case OP_EXTRACT_I1: - case OP_EXTRACT_U1: { - LLVMTypeRef t; + break; + case OP_EXTRACT_U2: + case OP_EXTRACT_U1: + zext = TRUE; + break; + default: + t = LLVMInt32Type (); + g_assert_not_reached (); + } + + lhs = LLVMBuildBitCast (builder, lhs, t, ""); + values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), ""); + if (zext) + values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), ""); + break; + } + + case OP_EXPAND_I1: + case OP_EXPAND_I2: + case OP_EXPAND_I4: + case OP_EXPAND_I8: + case OP_EXPAND_R4: + case OP_EXPAND_R8: { + LLVMTypeRef t = simd_op_to_llvm_type (ins->opcode); + LLVMValueRef mask [16], v; + + for (i = 0; i < 16; ++i) + mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); + + v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t)); + + values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMConstNull (t), v, LLVMConstInt (LLVMInt32Type (), 0, FALSE), ""); + values [ins->dreg] = LLVMBuildShuffleVector (builder, values [ins->dreg], LLVMGetUndef (t), LLVMConstVector (mask, LLVMGetVectorSize (t)), ""); + break; + } +#endif + + case OP_DUMMY_USE: + break; + + /* + * EXCEPTION HANDLING + */ + case OP_IMPLICIT_EXCEPTION: + /* This marks a place where an implicit exception can happen */ + if (bb->region != -1) + LLVM_FAILURE (ctx, "implicit-exception"); + break; + case OP_THROW: + case OP_RETHROW: { + MonoMethodSignature *throw_sig; + LLVMValueRef callee, arg; + gboolean rethrow = (ins->opcode == OP_RETHROW); + const char *icall_name; + + callee = rethrow ? ctx->lmodule->rethrow : ctx->lmodule->throw; + icall_name = rethrow ? "mono_arch_rethrow_exception" : "mono_arch_throw_exception"; + + if (!callee) { + throw_sig = mono_metadata_signature_alloc (mono_get_corlib (), 1); + throw_sig->ret = &mono_get_void_class ()->byval_arg; + throw_sig->params [0] = &mono_get_object_class ()->byval_arg; + if (cfg->compile_aot) { + callee = get_plt_entry (ctx, sig_to_llvm_sig (ctx, throw_sig), MONO_PATCH_INFO_INTERNAL_METHOD, icall_name); + } else { + callee = LLVMAddFunction (module, icall_name, sig_to_llvm_sig (ctx, throw_sig)); + +#ifdef TARGET_X86 + /* + * LLVM doesn't push the exception argument, so we need a different + * trampoline. + */ + LLVMAddGlobalMapping (ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, rethrow ? "llvm_rethrow_exception_trampoline" : "llvm_throw_exception_trampoline")); +#else + LLVMAddGlobalMapping (ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, icall_name)); +#endif + } + + mono_memory_barrier (); + if (rethrow) + ctx->lmodule->rethrow = callee; + else + ctx->lmodule->throw = callee; + } + arg = convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, &mono_get_object_class ()->byval_arg)); + emit_call (ctx, bb, &builder, callee, &arg, 1); + break; + } + case OP_CALL_HANDLER: { + /* + * We don't 'call' handlers, but instead simply branch to them. + * The code generated by ENDFINALLY will branch back to us. + */ + LLVMBasicBlockRef noex_bb; + GSList *bb_list; + BBInfo *info = &bblocks [ins->inst_target_bb->block_num]; + + bb_list = info->call_handler_return_bbs; + + /* + * Set the indicator variable for the finally clause. + */ + lhs = info->finally_ind; + g_assert (lhs); + LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs); + + /* Branch to the finally clause */ + LLVMBuildBr (builder, info->call_handler_target_bb); + + noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB"); + info->call_handler_return_bbs = g_slist_append_mempool (cfg->mempool, info->call_handler_return_bbs, noex_bb); + + builder = ctx->builder = create_builder (ctx); + LLVMPositionBuilderAtEnd (ctx->builder, noex_bb); + + bblocks [bb->block_num].end_bblock = noex_bb; + break; + } + case OP_START_HANDLER: { + break; + } + case OP_ENDFINALLY: { + LLVMBasicBlockRef resume_bb; + MonoBasicBlock *handler_bb; + LLVMValueRef val, switch_ins, callee; + GSList *bb_list; + BBInfo *info; + + handler_bb = g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region))); + g_assert (handler_bb); + info = &bblocks [handler_bb->block_num]; + lhs = info->finally_ind; + g_assert (lhs); + + bb_list = info->call_handler_return_bbs; + + resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB"); + + /* Load the finally variable */ + val = LLVMBuildLoad (builder, lhs, ""); + + /* Reset the variable */ + LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs); + + /* Branch to either resume_bb, or to the bblocks in bb_list */ + switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list)); + /* + * The other targets are added at the end to handle OP_CALL_HANDLER + * opcodes processed later. + */ + info->endfinally_switch_ins_list = g_slist_append_mempool (cfg->mempool, info->endfinally_switch_ins_list, switch_ins); + + builder = ctx->builder = create_builder (ctx); + LLVMPositionBuilderAtEnd (ctx->builder, resume_bb); + + if (ctx->cfg->compile_aot) { + callee = get_plt_entry (ctx, LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE), MONO_PATCH_INFO_INTERNAL_METHOD, "llvm_resume_unwind_trampoline"); + } else { + callee = LLVMGetNamedFunction (module, "llvm_resume_unwind_trampoline"); + } + LLVMBuildCall (builder, callee, NULL, 0, ""); + + LLVMBuildUnreachable (builder); + has_terminator = TRUE; + break; + } + default: { + char reason [128]; + + sprintf (reason, "opcode %s", mono_inst_name (ins->opcode)); + LLVM_FAILURE (ctx, reason); + break; + } + } + + /* Convert the value to the type required by phi nodes */ + if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && ctx->vreg_types [ins->dreg]) { + if (!values [ins->dreg]) + /* vtypes */ + values [ins->dreg] = addresses [ins->dreg]; + else + values [ins->dreg] = convert (ctx, values [ins->dreg], ctx->vreg_types [ins->dreg]); + } + + /* Add stores for volatile variables */ + if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins)) + emit_volatile_store (ctx, ins->dreg); + } + + if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) + LLVMBuildBr (builder, get_bb (ctx, bb->next_bb)); + + if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) + LLVMBuildRetVoid (builder); + + if (bb == cfg->bb_entry) + ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry)); + + return; + + FAILURE: + return; +} + +/* + * mono_llvm_check_method_supported: + * + * Do some quick checks to decide whenever cfg->method can be compiled by LLVM, to avoid + * compiling a method twice. + */ +void +mono_llvm_check_method_supported (MonoCompile *cfg) +{ + /* + MonoMethodHeader *header = cfg->header; + MonoExceptionClause *clause; + int i; + */ + + if (cfg->generic_sharing_context && !IS_LLVM_MONO_BRANCH) { + /* No way to obtain location info for this/rgctx */ + cfg->exception_message = g_strdup ("gshared"); + cfg->disable_llvm = TRUE; + } + + if (cfg->method->save_lmf) { + cfg->exception_message = g_strdup ("lmf"); + cfg->disable_llvm = TRUE; + } + + if (!LLVM_CHECK_VERSION (2, 8)) { + /* + * FIXME: LLLVM 2.6 no longer seems to generate correct exception info + * for JITted code. + */ + cfg->exception_message = g_strdup ("clauses"); + cfg->disable_llvm = TRUE; + } + +#if 0 + for (i = 0; i < header->num_clauses; ++i) { + clause = &header->clauses [i]; + + if (i > 0 && clause->try_offset <= header->clauses [i - 1].handler_offset + header->clauses [i - 1].handler_len) { + /* + * FIXME: Some tests still fail with nested clauses. + */ + cfg->exception_message = g_strdup ("nested clauses"); + cfg->disable_llvm = TRUE; + } + } +#endif + + /* FIXME: */ + if (cfg->method->dynamic) { + cfg->exception_message = g_strdup ("dynamic."); + cfg->disable_llvm = TRUE; + } +} + +/* + * mono_llvm_emit_method: + * + * Emit LLVM IL from the mono IL, and compile it to native code using LLVM. + */ +void +mono_llvm_emit_method (MonoCompile *cfg) +{ + EmitContext *ctx; + MonoMethodSignature *sig; + MonoBasicBlock *bb; + LLVMTypeRef method_type; + LLVMValueRef method = NULL; + char *method_name; + LLVMValueRef *values; + int i, max_block_num, bb_index; + gboolean last = FALSE; + GPtrArray *phi_values; + LLVMCallInfo *linfo; + GSList *l; + LLVMModuleRef module; + BBInfo *bblocks; + GPtrArray *bblock_list; + MonoMethodHeader *header; + MonoExceptionClause *clause; + LLVMSigInfo sinfo; + char **names; + + /* The code below might acquire the loader lock, so use it for global locking */ + mono_loader_lock (); + + /* Used to communicate with the callbacks */ + TlsSetValue (current_cfg_tls_id, cfg); + + ctx = g_new0 (EmitContext, 1); + ctx->cfg = cfg; + ctx->mempool = cfg->mempool; + + /* + * This maps vregs to the LLVM instruction defining them + */ + values = g_new0 (LLVMValueRef, cfg->next_vreg); + /* + * This maps vregs for volatile variables to the LLVM instruction defining their + * address. + */ + ctx->addresses = g_new0 (LLVMValueRef, cfg->next_vreg); + ctx->vreg_types = g_new0 (LLVMTypeRef, cfg->next_vreg); + ctx->vreg_cli_types = g_new0 (MonoType*, cfg->next_vreg); + phi_values = g_ptr_array_new (); + /* + * This signals whenever the vreg was defined by a phi node with no input vars + * (i.e. all its input bblocks end with NOT_REACHABLE). + */ + ctx->is_dead = g_new0 (gboolean, cfg->next_vreg); + /* Whenever the bblock is unreachable */ + ctx->unreachable = g_new0 (gboolean, cfg->max_block_num); - switch (ins->opcode) { - case OP_EXTRACT_R8: - t = LLVMVectorType (LLVMDoubleType (), 2); - break; - case OP_EXTRACT_I8: - t = LLVMVectorType (LLVMInt64Type (), 2); - break; - case OP_EXTRACT_I4: - t = LLVMVectorType (LLVMInt32Type (), 4); - break; - case OP_EXTRACT_I2: - case OP_EXTRACT_U2: - t = LLVMVectorType (LLVMInt16Type (), 8); - break; - case OP_EXTRACT_I1: - case OP_EXTRACT_U1: - t = LLVMVectorType (LLVMInt8Type (), 16); - break; - default: - t = LLVMInt32Type (); - g_assert_not_reached (); - } + bblock_list = g_ptr_array_new (); - lhs = LLVMBuildBitCast (builder, lhs, t, ""); - values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), ""); - break; + ctx->values = values; + ctx->region_to_handler = g_hash_table_new (NULL, NULL); + + if (cfg->compile_aot) { + ctx->lmodule = &aot_module; + method_name = mono_aot_get_method_name (cfg); + cfg->llvm_method_name = g_strdup (method_name); + } else { + init_jit_module (); + ctx->lmodule = &jit_module; + method_name = mono_method_full_name (cfg->method, TRUE); + } + + module = ctx->module = ctx->lmodule->module; + +#if 1 + { + static int count = 0; + count ++; + + if (getenv ("LLVM_COUNT")) { + if (count == atoi (getenv ("LLVM_COUNT"))) { + printf ("LAST: %s\n", mono_method_full_name (cfg->method, TRUE)); + fflush (stdout); + last = TRUE; } + if (count > atoi (getenv ("LLVM_COUNT"))) + LLVM_FAILURE (ctx, ""); + } + } #endif - case OP_DUMMY_USE: - break; + sig = mono_method_signature (cfg->method); + ctx->sig = sig; - /* - * EXCEPTION HANDLING - */ - case OP_IMPLICIT_EXCEPTION: - /* This marks a place where an implicit exception can happen */ - if (bb->region != -1) - LLVM_FAILURE (ctx, "implicit-exception"); - break; - case OP_THROW: { - MonoMethodSignature *throw_sig; - LLVMValueRef callee, arg; - - if (!ctx->lmodule->throw) { - throw_sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1); - throw_sig->ret = &mono_defaults.void_class->byval_arg; - throw_sig->params [0] = &mono_defaults.object_class->byval_arg; - if (cfg->compile_aot) { - callee = get_plt_entry (ctx, sig_to_llvm_sig (ctx, throw_sig, NULL), MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_exception"); - } else { - callee = LLVMAddFunction (module, "mono_arch_throw_exception", sig_to_llvm_sig (ctx, throw_sig, NULL)); + linfo = mono_arch_get_llvm_call_info (cfg, sig); + ctx->linfo = linfo; + CHECK_FAILURE (ctx); -#ifdef TARGET_X86 - /* - * LLVM doesn't push the exception argument, so we need a different - * trampoline. - */ - LLVMAddGlobalMapping (ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_llvm_throw_exception")); -#else - LLVMAddGlobalMapping (ee, callee, resolve_patch (cfg, MONO_PATCH_INFO_INTERNAL_METHOD, "mono_arch_throw_exception")); + if (cfg->rgctx_var) { + if (IS_LLVM_MONO_BRANCH) + linfo->rgctx_arg = TRUE; + else + LLVM_FAILURE (ctx, "rgctx arg"); + } + method_type = sig_to_llvm_sig_full (ctx, sig, linfo, &sinfo); + CHECK_FAILURE (ctx); + + /* + * This maps parameter indexes in the original signature to the indexes in + * the LLVM signature. + */ + ctx->pindexes = sinfo.pindexes; + + method = LLVMAddFunction (module, method_name, method_type); + ctx->lmethod = method; + +#ifdef LLVM_MONO_BRANCH + LLVMSetFunctionCallConv (method, LLVMMono1CallConv); #endif - } + LLVMSetLinkage (method, LLVMPrivateLinkage); - mono_memory_barrier (); - ctx->lmodule->throw = callee; - } - arg = convert (ctx, values [ins->sreg1], type_to_llvm_type (ctx, &mono_defaults.object_class->byval_arg)); - emit_call (ctx, bb, &builder, ctx->lmodule->throw, &arg, 1); - break; - } - case OP_CALL_HANDLER: { - /* - * We don't 'call' handlers, but instead simply branch to them. - * The code generated by ENDFINALLY will branch back to us. - */ - LLVMBasicBlockRef finally_bb, noex_bb; - GSList *bb_list; + if (cfg->compile_aot) { + LLVMSetLinkage (method, LLVMInternalLinkage); + LLVMSetVisibility (method, LLVMHiddenVisibility); + } else { + LLVMSetLinkage (method, LLVMPrivateLinkage); + } + + if (cfg->method->save_lmf) + LLVM_FAILURE (ctx, "lmf"); - finally_bb = get_bb (ctx, ins->inst_target_bb); + if (sig->pinvoke) + LLVM_FAILURE (ctx, "pinvoke signature"); - bb_list = bblocks [ins->inst_target_bb->block_num].call_handler_return_bbs; + header = cfg->header; + for (i = 0; i < header->num_clauses; ++i) { + clause = &header->clauses [i]; + if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY && clause->flags != MONO_EXCEPTION_CLAUSE_NONE) + LLVM_FAILURE (ctx, "non-finally/catch clause."); + } - /* - * Set the indicator variable for the finally clause. - */ - lhs = bblocks [ins->inst_target_bb->block_num].finally_ind; - g_assert (lhs); - LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), g_slist_length (bb_list) + 1, FALSE), lhs); - - /* Branch to the finally clause */ - LLVMBuildBr (builder, finally_bb); + if (linfo->rgctx_arg) { + ctx->rgctx_arg = LLVMGetParam (method, sinfo.rgctx_arg_pindex); + /* + * We mark the rgctx parameter with the inreg attribute, which is mapped to + * MONO_ARCH_RGCTX_REG in the Mono calling convention in llvm, i.e. + * CC_X86_64_Mono in X86CallingConv.td. + */ + LLVMAddAttribute (ctx->rgctx_arg, LLVMInRegAttribute); + LLVMSetValueName (ctx->rgctx_arg, "rgctx"); + } + if (cfg->vret_addr) { + values [cfg->vret_addr->dreg] = LLVMGetParam (method, sinfo.vret_arg_pindex); + LLVMSetValueName (values [cfg->vret_addr->dreg], "vret"); + } + if (sig->hasthis) { + values [cfg->args [0]->dreg] = LLVMGetParam (method, sinfo.this_arg_pindex); + LLVMSetValueName (values [cfg->args [0]->dreg], "this"); + } - noex_bb = gen_bb (ctx, "CALL_HANDLER_CONT_BB"); - // FIXME: Use a mempool - bblocks [ins->inst_target_bb->block_num].call_handler_return_bbs = g_slist_append (bblocks [ins->inst_target_bb->block_num].call_handler_return_bbs, noex_bb); + names = g_new (char *, sig->param_count); + mono_method_get_param_names (cfg->method, (const char **) names); - builder = ctx->builder = create_builder (ctx); - LLVMPositionBuilderAtEnd (ctx->builder, noex_bb); + for (i = 0; i < sig->param_count; ++i) { + char *name; + + values [cfg->args [i + sig->hasthis]->dreg] = LLVMGetParam (method, sinfo.pindexes [i]); + if (names [i] && names [i][0] != '\0') + name = g_strdup_printf ("arg_%s", names [i]); + else + name = g_strdup_printf ("arg_%d", i); + LLVMSetValueName (values [cfg->args [i + sig->hasthis]->dreg], name); + g_free (name); + if (linfo->args [i + sig->hasthis].storage == LLVMArgVtypeByVal) + LLVMAddAttribute (LLVMGetParam (method, sinfo.pindexes [i]), LLVMByValAttribute); + } + g_free (names); - bblocks [bb->block_num].end_bblock = noex_bb; - break; - } - case OP_START_HANDLER: { - break; - } - case OP_ENDFINALLY: { - LLVMBasicBlockRef resume_bb; - MonoBasicBlock *handler_bb; - LLVMValueRef val, switch_ins; - GSList *bb_list; + max_block_num = 0; + for (bb = cfg->bb_entry; bb; bb = bb->next_bb) + max_block_num = MAX (max_block_num, bb->block_num); + ctx->bblocks = bblocks = g_new0 (BBInfo, max_block_num + 1); + + /* Add branches between non-consecutive bblocks */ + for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { + if (bb->last_ins && MONO_IS_COND_BRANCH_OP (bb->last_ins) && + bb->next_bb != bb->last_ins->inst_false_bb) { + + MonoInst *inst = mono_mempool_alloc0 (cfg->mempool, sizeof (MonoInst)); + inst->opcode = OP_BR; + inst->inst_target_bb = bb->last_ins->inst_false_bb; + mono_bblock_add_inst (bb, inst); + } + } + + /* + * The INDIRECT flag added by OP_LDADDR inhibits optimizations, even if the LDADDR + * was later optimized away, so clear these flags, and add them back for the still + * present OP_LDADDR instructions. + */ + for (i = 0; i < cfg->next_vreg; ++i) { + MonoInst *ins; + + ins = get_vreg_to_inst (cfg, i); + if (ins && ins != cfg->rgctx_var) + ins->flags &= ~MONO_INST_INDIRECT; + } - handler_bb = g_hash_table_lookup (ctx->region_to_handler, GUINT_TO_POINTER (mono_get_block_region_notry (cfg, bb->region))); - g_assert (handler_bb); - lhs = bblocks [handler_bb->block_num].finally_ind; - g_assert (lhs); + /* + * Make a first pass over the code to precreate PHI nodes/set INDIRECT flags. + */ + for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { + MonoInst *ins; + LLVMBuilderRef builder; + char *dname; + char dname_buf[128]; - bb_list = bblocks [handler_bb->block_num].call_handler_return_bbs; + builder = create_builder (ctx); - resume_bb = gen_bb (ctx, "ENDFINALLY_RESUME_BB"); + for (ins = bb->code; ins; ins = ins->next) { + switch (ins->opcode) { + case OP_PHI: + case OP_FPHI: + case OP_VPHI: + case OP_XPHI: { + LLVMTypeRef phi_type = llvm_type_to_stack_type (type_to_llvm_type (ctx, &ins->klass->byval_arg)); - /* Load the finally variable */ - val = LLVMBuildLoad (builder, lhs, ""); + CHECK_FAILURE (ctx); - /* Reset the variable */ - LLVMBuildStore (builder, LLVMConstInt (LLVMInt32Type (), 0, FALSE), lhs); + if (ins->opcode == OP_VPHI) { + /* Treat valuetype PHI nodes as operating on the address itself */ + g_assert (ins->klass); + phi_type = LLVMPointerType (type_to_llvm_type (ctx, &ins->klass->byval_arg), 0); + } - /* Branch to either resume_bb, or to the bblocks in bb_list */ - switch_ins = LLVMBuildSwitch (builder, val, resume_bb, g_slist_length (bb_list)); /* - * The other targets are added at the end to handle OP_CALL_HANDLER - * opcodes processed later. + * Have to precreate these, as they can be referenced by + * earlier instructions. */ - bblocks [handler_bb->block_num].endfinally_switch = switch_ins; - /* - for (i = 0; i < g_slist_length (bb_list); ++i) - LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data); - */ + sprintf (dname_buf, "t%d", ins->dreg); + dname = dname_buf; + values [ins->dreg] = LLVMBuildPhi (builder, phi_type, dname); - builder = ctx->builder = create_builder (ctx); - LLVMPositionBuilderAtEnd (ctx->builder, resume_bb); + if (ins->opcode == OP_VPHI) + ctx->addresses [ins->dreg] = values [ins->dreg]; - LLVMBuildCall (builder, LLVMGetNamedFunction (module, "mono_resume_unwind"), NULL, 0, ""); - LLVMBuildUnreachable (builder); - has_terminator = TRUE; - break; - } - default: { - char reason [128]; + g_ptr_array_add (phi_values, values [ins->dreg]); - sprintf (reason, "opcode %s", mono_inst_name (ins->opcode)); - LLVM_FAILURE (ctx, reason); + /* + * Set the expected type of the incoming arguments since these have + * to have the same type. + */ + for (i = 0; i < ins->inst_phi_args [0]; i++) { + int sreg1 = ins->inst_phi_args [i + 1]; + + if (sreg1 != -1) + ctx->vreg_types [sreg1] = phi_type; + } + break; + } + case OP_LDADDR: + ((MonoInst*)ins->inst_p0)->flags |= MONO_INST_INDIRECT; + break; + default: break; } - } - - /* Convert the value to the type required by phi nodes */ - if (spec [MONO_INST_DEST] != ' ' && !MONO_IS_STORE_MEMBASE (ins) && vreg_types [ins->dreg]) { - if (!values [ins->dreg]) - /* vtypes */ - values [ins->dreg] = addresses [ins->dreg]; - else - values [ins->dreg] = convert (ctx, values [ins->dreg], vreg_types [ins->dreg]); - } + } + } - /* Add stores for volatile variables */ - if (spec [MONO_INST_DEST] != ' ' && spec [MONO_INST_DEST] != 'v' && !MONO_IS_STORE_MEMBASE (ins)) - emit_volatile_store (ctx, ins->dreg); + /* + * Create an ordering for bblocks, use the depth first order first, then + * put the exception handling bblocks last. + */ + for (bb_index = 0; bb_index < cfg->num_bblocks; ++bb_index) { + bb = cfg->bblocks [bb_index]; + if (!(bb->region != -1 && !MONO_BBLOCK_IS_IN_REGION (bb, MONO_REGION_TRY))) { + g_ptr_array_add (bblock_list, bb); + bblocks [bb->block_num].added = TRUE; } + } + + for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { + if (!bblocks [bb->block_num].added) + g_ptr_array_add (bblock_list, bb); + } - if (!has_terminator && bb->next_bb && (bb == cfg->bb_entry || bb->in_count > 0)) - LLVMBuildBr (builder, get_bb (ctx, bb->next_bb)); + /* + * Second pass: generate code. + */ + for (bb_index = 0; bb_index < bblock_list->len; ++bb_index) { + bb = g_ptr_array_index (bblock_list, bb_index); - if (bb == cfg->bb_exit && sig->ret->type == MONO_TYPE_VOID) - LLVMBuildRetVoid (builder); + if (!(bb == cfg->bb_entry || bb->in_count > 0)) + continue; - if (bb == cfg->bb_entry) - ctx->last_alloca = LLVMGetLastInstruction (get_bb (ctx, cfg->bb_entry)); + process_bb (ctx, bb); + CHECK_FAILURE (ctx); } /* Add incoming phi values */ @@ -3431,21 +4043,28 @@ mono_llvm_emit_method (MonoCompile *cfg) in_bb = get_end_bb (ctx, node->in_bb); - if (unreachable [node->in_bb->block_num]) + if (ctx->unreachable [node->in_bb->block_num]) continue; g_assert (values [sreg1]); - g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg])); - LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1); + if (phi->opcode == OP_VPHI) { + g_assert (LLVMTypeOf (ctx->addresses [sreg1]) == LLVMTypeOf (values [phi->dreg])); + LLVMAddIncoming (values [phi->dreg], &ctx->addresses [sreg1], &in_bb, 1); + } else { + g_assert (LLVMTypeOf (values [sreg1]) == LLVMTypeOf (values [phi->dreg])); + LLVMAddIncoming (values [phi->dreg], &values [sreg1], &in_bb, 1); + } } } /* Create the SWITCH statements for ENDFINALLY instructions */ for (bb = cfg->bb_entry; bb; bb = bb->next_bb) { - if (bblocks [bb->block_num].endfinally_switch) { - LLVMValueRef switch_ins = bblocks [bb->block_num].endfinally_switch; - GSList *bb_list = bblocks [bb->block_num].call_handler_return_bbs; + BBInfo *info = &bblocks [bb->block_num]; + GSList *l; + for (l = info->endfinally_switch_ins_list; l; l = l->next) { + LLVMValueRef switch_ins = l->data; + GSList *bb_list = info->call_handler_return_bbs; for (i = 0; i < g_slist_length (bb_list); ++i) LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i + 1, FALSE), g_slist_nth (bb_list, i)->data); @@ -3459,13 +4078,6 @@ mono_llvm_emit_method (MonoCompile *cfg) if (cfg->compile_aot) { /* Don't generate native code, keep the LLVM IR */ - - /* Can't delete the method if it has an alias, so only add it if successful */ - if (debug_name) { - debug_alias = LLVMAddAlias (module, LLVMTypeOf (method), method, debug_name); - LLVMSetVisibility (debug_alias, LLVMHiddenVisibility); - } - if (cfg->compile_aot && cfg->verbose_level) printf ("%s emitted as %s\n", mono_method_full_name (cfg->method, TRUE), method_name); @@ -3507,11 +4119,12 @@ mono_llvm_emit_method (MonoCompile *cfg) CLEANUP: g_free (values); - g_free (addresses); - g_free (vreg_types); - g_free (vreg_cli_types); - g_free (pindexes); - g_free (debug_name); + g_free (ctx->addresses); + g_free (ctx->vreg_types); + g_free (ctx->vreg_cli_types); + g_free (ctx->pindexes); + g_free (ctx->is_dead); + g_free (ctx->unreachable); g_ptr_array_free (phi_values, TRUE); g_free (ctx->bblocks); g_hash_table_destroy (ctx->region_to_handler); @@ -3628,8 +4241,9 @@ exception_cb (void *data) { MonoCompile *cfg; MonoJitExceptionInfo *ei; - guint32 ei_len, i; + guint32 ei_len, i, j, nested_len, nindex; gpointer *type_info; + int this_reg, this_offset; cfg = TlsGetValue (current_cfg_tls_id); g_assert (cfg); @@ -3640,20 +4254,68 @@ exception_cb (void *data) * An alternative would be to save it directly, and modify our unwinder to work * with it. */ - cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info); + cfg->encoded_unwind_ops = mono_unwind_decode_fde ((guint8*)data, &cfg->encoded_unwind_ops_len, NULL, &ei, &ei_len, &type_info, &this_reg, &this_offset); + + /* Count nested clauses */ + nested_len = 0; + for (i = 0; i < ei_len; ++i) { + for (j = 0; j < ei_len; ++j) { + gint32 cindex1 = *(gint32*)type_info [i]; + MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1]; + gint32 cindex2 = *(gint32*)type_info [j]; + MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2]; + + if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) { + nested_len ++; + } + } + } - cfg->llvm_ex_info = mono_mempool_alloc0 (cfg->mempool, ei_len * sizeof (MonoJitExceptionInfo)); - cfg->llvm_ex_info_len = ei_len; + cfg->llvm_ex_info = mono_mempool_alloc0 (cfg->mempool, (ei_len + nested_len) * sizeof (MonoJitExceptionInfo)); + cfg->llvm_ex_info_len = ei_len + nested_len; memcpy (cfg->llvm_ex_info, ei, ei_len * sizeof (MonoJitExceptionInfo)); /* Fill the rest of the information from the type info */ for (i = 0; i < ei_len; ++i) { - MonoExceptionClause *clause = type_info [i]; + gint32 clause_index = *(gint32*)type_info [i]; + MonoExceptionClause *clause = &cfg->header->clauses [clause_index]; cfg->llvm_ex_info [i].flags = clause->flags; cfg->llvm_ex_info [i].data.catch_class = clause->data.catch_class; } + /* + * For nested clauses, the LLVM produced exception info associates the try interval with + * the innermost handler, while mono expects it to be associated with all nesting clauses. + */ + /* FIXME: These should be order with the normal clauses */ + nindex = ei_len; + for (i = 0; i < ei_len; ++i) { + for (j = 0; j < ei_len; ++j) { + gint32 cindex1 = *(gint32*)type_info [i]; + MonoExceptionClause *clause1 = &cfg->header->clauses [cindex1]; + gint32 cindex2 = *(gint32*)type_info [j]; + MonoExceptionClause *clause2 = &cfg->header->clauses [cindex2]; + + if (cindex1 != cindex2 && clause1->try_offset >= clause2->try_offset && clause1->handler_offset <= clause2->handler_offset) { + /* + * The try interval comes from the nested clause, everything else from the + * nesting clause. + */ + memcpy (&cfg->llvm_ex_info [nindex], &cfg->llvm_ex_info [j], sizeof (MonoJitExceptionInfo)); + cfg->llvm_ex_info [nindex].try_start = cfg->llvm_ex_info [i].try_start; + cfg->llvm_ex_info [nindex].try_end = cfg->llvm_ex_info [i].try_end; + nindex ++; + } + } + } + g_assert (nindex == ei_len + nested_len); + cfg->llvm_this_reg = this_reg; + cfg->llvm_this_offset = this_offset; + + /* type_info [i] is cfg mempool allocated, no need to free it */ + g_free (ei); + g_free (type_info); } static void @@ -3661,15 +4323,30 @@ add_intrinsics (LLVMModuleRef module) { /* Emit declarations of instrinsics */ { - LLVMTypeRef memset_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type () }; + LLVMTypeRef memset_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMInt8Type (), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () }; - LLVMAddFunction (module, "llvm.memset.i32", LLVMFunctionType (LLVMVoidType (), memset_params, 4, FALSE)); + if (LLVM_CHECK_VERSION(2, 8)) { + memset_param_count = 5; + memset_func_name = "llvm.memset.p0i8.i32"; + } else { + memset_param_count = 4; + memset_func_name = "llvm.memset.i32"; + } + LLVMAddFunction (module, memset_func_name, LLVMFunctionType (LLVMVoidType (), memset_params, memset_param_count, FALSE)); } { - LLVMTypeRef memcpy_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type () }; + LLVMTypeRef memcpy_params [] = { LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type (), LLVMInt32Type (), LLVMInt1Type () }; + + if (LLVM_CHECK_VERSION(2, 8)) { + memcpy_param_count = 5; + memcpy_func_name = "llvm.memcpy.p0i8.p0i8.i32"; + } else { + memcpy_param_count = 4; + memcpy_func_name = "llvm.memcpy.i32"; + } - LLVMAddFunction (module, "llvm.memcpy.i32", LLVMFunctionType (LLVMVoidType (), memcpy_params, 4, FALSE)); + LLVMAddFunction (module, memcpy_func_name, LLVMFunctionType (LLVMVoidType (), memcpy_params, memcpy_param_count, FALSE)); } { @@ -3732,16 +4409,29 @@ add_intrinsics (LLVMModuleRef module) /* EH intrinsics */ { LLVMTypeRef arg_types [2]; + LLVMTypeRef ret_type; arg_types [0] = LLVMPointerType (LLVMInt8Type (), 0); arg_types [1] = LLVMPointerType (LLVMInt8Type (), 0); - LLVMAddFunction (module, "llvm.eh.selector", LLVMFunctionType (LLVMInt32Type (), arg_types, 2, TRUE)); + if (LLVM_CHECK_VERSION(2, 8)) { + eh_selector_name = "llvm.eh.selector"; + ret_type = LLVMInt32Type (); + } else { + if (SIZEOF_VOID_P == 8) { + eh_selector_name = "llvm.eh.selector.i64"; + ret_type = LLVMInt64Type (); + } else { + eh_selector_name = "llvm.eh.selector.i32"; + ret_type = LLVMInt32Type (); + } + } + LLVMAddFunction (module, eh_selector_name, LLVMFunctionType (ret_type, arg_types, 2, TRUE)); LLVMAddFunction (module, "llvm.eh.exception", LLVMFunctionType (LLVMPointerType (LLVMInt8Type (), 0), NULL, 0, FALSE)); LLVMAddFunction (module, "mono_personality", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE)); - LLVMAddFunction (module, "mono_resume_unwind", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE)); + LLVMAddFunction (module, "llvm_resume_unwind_trampoline", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE)); } /* SSE intrinsics */ @@ -3751,32 +4441,64 @@ add_intrinsics (LLVMModuleRef module) vector_type = LLVMVectorType (LLVMInt32Type (), 4); arg_types [0] = vector_type; arg_types [1] = vector_type; - LLVMAddFunction (module, "llvm.x86.sse41.pminud", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); - LLVMAddFunction (module, "llvm.x86.sse41.pmaxud", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse41.pminud", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse41.pmaxud", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pcmpeq.d", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); vector_type = LLVMVectorType (LLVMInt16Type (), 8); arg_types [0] = vector_type; arg_types [1] = vector_type; - LLVMAddFunction (module, "llvm.x86.sse41.pminuw", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); - LLVMAddFunction (module, "llvm.x86.sse41.pmaxuw", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse41.pminuw", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pmins.w", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse41.pmaxuw", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pcmpeq.w", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); vector_type = LLVMVectorType (LLVMInt8Type (), 16); arg_types [0] = vector_type; arg_types [1] = vector_type; - LLVMAddFunction (module, "llvm.x86.sse41.pminub", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); - LLVMAddFunction (module, "llvm.x86.sse41.pmaxub", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pminu.b", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pmaxu.b", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pcmpeq.b", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.pcmpgt.b", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + + vector_type = LLVMVectorType (LLVMInt64Type (), 2); + arg_types [0] = vector_type; + arg_types [1] = vector_type; + LLVMAddFunction (module, "llvm.x86.sse41.pcmpeqq", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); vector_type = LLVMVectorType (LLVMDoubleType (), 2); arg_types [0] = vector_type; arg_types [1] = vector_type; - LLVMAddFunction (module, "llvm.x86.sse2.min.pd", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); - LLVMAddFunction (module, "llvm.x86.sse2.max.pd", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.min.pd", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.max.pd", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); vector_type = LLVMVectorType (LLVMFloatType (), 4); arg_types [0] = vector_type; arg_types [1] = vector_type; - LLVMAddFunction (module, "llvm.x86.sse2.min.ps", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); - LLVMAddFunction (module, "llvm.x86.sse2.max.ps", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.min.ps", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + LLVMAddFunction (module, "llvm.x86.sse2.max.ps", LLVMFunctionType (vector_type, arg_types, 2, FALSE)); + } + + /* Load/Store intrinsics */ + if (IS_LLVM_MONO_BRANCH) { + LLVMTypeRef arg_types [5]; + int i; + char name [128]; + + for (i = 1; i <= 8; i *= 2) { + arg_types [0] = LLVMPointerType (LLVMIntType (i * 8), 0); + arg_types [1] = LLVMInt32Type (); + arg_types [2] = LLVMInt1Type (); + sprintf (name, "llvm.mono.load.i%d.p0i%d", i * 8, i * 8); + LLVMAddFunction (module, name, LLVMFunctionType (LLVMIntType (i * 8), arg_types, 3, FALSE)); + + arg_types [0] = LLVMIntType (i * 8); + arg_types [1] = LLVMPointerType (LLVMIntType (i * 8), 0); + arg_types [2] = LLVMInt32Type (); + arg_types [3] = LLVMInt1Type (); + sprintf (name, "llvm.mono.store.i%d.p0i%d", i * 8, i * 8); + LLVMAddFunction (module, name, LLVMFunctionType (LLVMVoidType (), arg_types, 4, FALSE)); + } } } @@ -3784,6 +4506,22 @@ void mono_llvm_init (void) { current_cfg_tls_id = TlsAlloc (); +} + +static void +init_jit_module (void) +{ + MonoJitICallInfo *info; + + if (jit_module_inited) + return; + + mono_loader_lock (); + + if (jit_module_inited) { + mono_loader_unlock (); + return; + } jit_module.module = LLVMModuleCreateWithName ("mono"); @@ -3793,15 +4531,28 @@ mono_llvm_init (void) jit_module.llvm_types = g_hash_table_new (NULL, NULL); - LLVMAddGlobalMapping (ee, LLVMGetNamedFunction (jit_module.module, "mono_resume_unwind"), mono_resume_unwind); + info = mono_find_jit_icall_by_name ("llvm_resume_unwind_trampoline"); + g_assert (info); + LLVMAddGlobalMapping (ee, LLVMGetNamedFunction (jit_module.module, "llvm_resume_unwind_trampoline"), (void*)info->func); + + jit_module_inited = TRUE; + + mono_loader_unlock (); } void mono_llvm_cleanup (void) { - mono_llvm_dispose_ee (ee); + if (ee) + mono_llvm_dispose_ee (ee); + + if (jit_module.llvm_types) + g_hash_table_destroy (jit_module.llvm_types); - g_hash_table_destroy (jit_module.llvm_types); + if (aot_module.module) + LLVMDisposeModule (aot_module.module); + + LLVMContextDispose (LLVMGetGlobalContext ()); } void @@ -3810,6 +4561,8 @@ mono_llvm_create_aot_module (const char *got_symbol) /* Delete previous module */ if (aot_module.plt_entries) g_hash_table_destroy (aot_module.plt_entries); + if (aot_module.module) + LLVMDisposeModule (aot_module.module); memset (&aot_module, 0, sizeof (aot_module)); @@ -3832,15 +4585,6 @@ mono_llvm_create_aot_module (const char *got_symbol) LLVMSetInitializer (aot_module.got_var, LLVMConstNull (got_type)); } - /* Add a method to generate the 'methods' symbol needed by the AOT compiler */ - { - LLVMValueRef methods_method = LLVMAddFunction (aot_module.module, "methods", LLVMFunctionType (LLVMVoidType (), NULL, 0, FALSE)); - LLVMBasicBlockRef bb = LLVMAppendBasicBlock (methods_method, "BB_ENTRY"); - LLVMBuilderRef builder = LLVMCreateBuilder (); - LLVMPositionBuilderAtEnd (builder, bb); - LLVMBuildRetVoid (builder); - } - /* Add a dummy personality function */ { LLVMBasicBlockRef lbb; @@ -3881,6 +4625,9 @@ mono_llvm_emit_aot_module (const char *filename, int got_size) mark_as_used (aot_module.module, real_got); + /* Delete the dummy got so it doesn't become a global */ + LLVMDeleteGlobal (aot_module.got_var); + #if 0 { char *verifier_err; @@ -3899,9 +4646,6 @@ mono_llvm_emit_aot_module (const char *filename, int got_size) - Emit LLVM IR from the mono IR using the LLVM C API. - The original arch specific code remains, so we can fall back to it if we run into something we can't handle. - FIXME: - - llvm's PrettyStackTrace class seems to register a signal handler which screws - up our GC. Also, it calls sigaction () a _lot_ of times instead of just once. */ /* @@ -3995,9 +4739,7 @@ mono_llvm_emit_aot_module (const char *filename, int got_size) /* FIXME: Normalize some aspects of the mono IR to allow easier translation, like: * - each bblock should end with a branch * - setting the return value, making cfg->ret non-volatile - * - merge some changes back to HEAD, to reduce the differences. * - avoid some transformations in the JIT which make it harder for us to generate * code. - * - fix memory leaks. * - use pointer types to help optimizations. */