#include <mono/metadata/marshal.h>
#include <mono/metadata/tabledefs.h>
#include <mono/utils/mono-counters.h>
-
-#ifdef HAVE_VALGRIND_MEMCHECK_H
-#include <valgrind/memcheck.h>
-#endif
+#include <mono/utils/mono-error-internals.h>
#include "mini.h"
#include "debug-mini.h"
guint8* mono_trampoline_code [MONO_TRAMPOLINE_NUM];
static GHashTable *class_init_hash_addr = NULL;
-static GHashTable *delegate_trampoline_hash_addr = NULL;
static GHashTable *rgctx_lazy_fetch_trampoline_hash = NULL;
static GHashTable *rgctx_lazy_fetch_trampoline_hash_addr = NULL;
+static guint32 trampoline_calls;
#define mono_trampolines_lock() EnterCriticalSection (&trampolines_mutex)
#define mono_trampolines_unlock() LeaveCriticalSection (&trampolines_mutex)
static CRITICAL_SECTION trampolines_mutex;
static gpointer
-get_unbox_trampoline (MonoGenericSharingContext *gsctx, MonoMethod *m, gpointer addr)
+get_unbox_trampoline (MonoGenericSharingContext *gsctx, MonoMethod *m, gpointer addr, gboolean need_rgctx_tramp)
+{
+ if (mono_aot_only) {
+ if (need_rgctx_tramp)
+ /*
+ * The unbox trampolines call the method directly, so need to add
+ * an rgctx tramp before them.
+ */
+ return mono_create_static_rgctx_trampoline (m, mono_aot_get_unbox_trampoline (m));
+ else
+ return mono_aot_get_unbox_trampoline (m);
+ } else {
+ return mono_arch_get_unbox_trampoline (gsctx, m, addr);
+ }
+}
+
+#ifdef MONO_ARCH_HAVE_STATIC_RGCTX_TRAMPOLINE
+
+typedef struct {
+ MonoMethod *m;
+ gpointer addr;
+} RgctxTrampInfo;
+
+static gint
+rgctx_tramp_info_equal (gconstpointer ka, gconstpointer kb)
{
+ const RgctxTrampInfo *i1 = ka;
+ const RgctxTrampInfo *i2 = kb;
+
+ if (i1->m == i2->m && i1->addr == i2->addr)
+ return 1;
+ else
+ return 0;
+}
+
+static guint
+rgctx_tramp_info_hash (gconstpointer data)
+{
+ const RgctxTrampInfo *info = data;
+
+ return GPOINTER_TO_UINT (info->m) ^ GPOINTER_TO_UINT (info->addr);
+}
+
+/*
+ * mono_create_static_rgctx_trampoline:
+ *
+ * Return a static rgctx trampoline for M which branches to ADDR which should
+ * point to the compiled code of M.
+ *
+ * Static rgctx trampolines are used when a shared generic method which doesn't
+ * have a this argument is called indirectly, ie. from code which can't pass in
+ * the rgctx argument. The trampoline sets the rgctx argument and jumps to the
+ * methods code. These trampolines are similar to the unbox trampolines, they
+ * perform the same task as the static rgctx wrappers, but they are smaller/faster,
+ * and can be made to work with full AOT.
+ * On PPC addr should be an ftnptr and the return value is an ftnptr too.
+ */
+gpointer
+mono_create_static_rgctx_trampoline (MonoMethod *m, gpointer addr)
+{
+ gpointer ctx;
+ gpointer res;
+ MonoDomain *domain;
+ RgctxTrampInfo tmp_info;
+ RgctxTrampInfo *info;
+
+#ifdef PPC_USES_FUNCTION_DESCRIPTOR
+ g_assert (((gpointer*)addr) [2] == 0);
+#endif
+
+ if (mini_method_get_context (m)->method_inst)
+ ctx = mono_method_lookup_rgctx (mono_class_vtable (mono_domain_get (), m->klass), mini_method_get_context (m)->method_inst);
+ else
+ ctx = mono_class_vtable (mono_domain_get (), m->klass);
+
+ domain = mono_domain_get ();
+
+ /*
+ * In the AOT case, addr might point to either the method, or to an unbox trampoline,
+ * so make the hash keyed on the m+addr pair.
+ */
+ mono_domain_lock (domain);
+ if (!domain_jit_info (domain)->static_rgctx_trampoline_hash)
+ domain_jit_info (domain)->static_rgctx_trampoline_hash = g_hash_table_new (rgctx_tramp_info_hash, rgctx_tramp_info_equal);
+ tmp_info.m = m;
+ tmp_info.addr = addr;
+ res = g_hash_table_lookup (domain_jit_info (domain)->static_rgctx_trampoline_hash,
+ &tmp_info);
+ mono_domain_unlock (domain);
+ if (res)
+ return res;
+
if (mono_aot_only)
- return mono_aot_get_unbox_trampoline (m);
+ res = mono_aot_get_static_rgctx_trampoline (ctx, addr);
else
- return mono_arch_get_unbox_trampoline (gsctx, m, addr);
+ res = mono_arch_get_static_rgctx_trampoline (m, ctx, addr);
+
+ mono_domain_lock (domain);
+ /* Duplicates inserted while we didn't hold the lock are OK */
+ info = mono_domain_alloc (domain, sizeof (RgctxTrampInfo));
+ info->m = m;
+ info->addr = addr;
+ g_hash_table_insert (domain_jit_info (domain)->static_rgctx_trampoline_hash, info, res);
+ mono_domain_unlock (domain);
+
+ return res;
+}
+#else
+gpointer
+mono_create_static_rgctx_trampoline (MonoMethod *m, gpointer addr)
+{
+ /*
+ * This shouldn't happen as all arches which support generic sharing support
+ * static rgctx trampolines as well.
+ */
+ g_assert_not_reached ();
+}
+#endif
+
+gpointer*
+mono_get_vcall_slot_addr (guint8* code, mgreg_t *regs)
+{
+ gpointer vt;
+ int displacement;
+ vt = mono_arch_get_vcall_slot (code, regs, &displacement);
+ if (!vt)
+ return NULL;
+ return (gpointer*)((char*)vt + displacement);
}
#ifdef MONO_ARCH_HAVE_IMT
static gpointer*
-mono_convert_imt_slot_to_vtable_slot (gpointer* slot, gpointer *regs, guint8 *code, MonoMethod *method, MonoMethod **impl_method)
+mono_convert_imt_slot_to_vtable_slot (gpointer* slot, mgreg_t *regs, guint8 *code, MonoMethod *method, MonoMethod **impl_method, gboolean *need_rgctx_tramp, gboolean *variance_used)
{
- MonoGenericSharingContext *gsctx = mono_get_generic_context_from_code (code);
- MonoObject *this_argument = mono_arch_find_this_argument (regs, method, gsctx);
+ MonoObject *this_argument = mono_arch_get_this_arg_from_call (NULL, mono_method_signature (method), regs, code);
MonoVTable *vt = this_argument->vtable;
int displacement = slot - ((gpointer*)vt);
int interface_offset;
int imt_slot = MONO_IMT_SIZE + displacement;
- interface_offset = mono_class_interface_offset (vt->klass, imt_method->klass);
+ /*This has to be variance aware since imt_method can be from an interface that vt->klass doesn't directly implement*/
+ interface_offset = mono_class_interface_offset_with_variance (vt->klass, imt_method->klass, variance_used);
if (interface_offset < 0) {
- g_print ("%s doesn't implement interface %s\n", mono_type_get_name_full (&vt->klass->byval_arg, 0), mono_type_get_name_full (&imt_method->klass->byval_arg, 0));
- g_assert_not_reached ();
+ g_error ("%s doesn't implement interface %s\n", mono_type_get_name_full (&vt->klass->byval_arg, 0), mono_type_get_name_full (&imt_method->klass->byval_arg, 0));
}
mono_vtable_build_imt_slot (vt, mono_method_get_imt_slot (imt_method));
context.method_inst = ((MonoMethodInflated*)imt_method)->context.method_inst;
impl = mono_class_inflate_generic_method (impl, &context);
} else {
- impl = mono_class_get_vtable_entry (vt->klass, interface_offset + imt_method->slot);
+ impl = mono_class_get_vtable_entry (vt->klass, interface_offset + mono_method_get_vtable_slot (imt_method));
}
+ if (mono_method_needs_static_rgctx_invoke (impl, FALSE))
+ *need_rgctx_tramp = TRUE;
+
*impl_method = impl;
#if DEBUG_IMT
printf ("mono_convert_imt_slot_to_vtable_slot: method = %s.%s.%s, imt_method = %s.%s.%s\n",
}
g_assert (imt_slot < MONO_IMT_SIZE);
if (vt->imt_collisions_bitmap & (1 << imt_slot)) {
- int vtable_offset = interface_offset + mono_method_get_vtable_index (imt_method);
- gpointer *vtable_slot = & (vt->vtable [vtable_offset]);
+ int slot = mono_method_get_vtable_index (imt_method);
+ int vtable_offset;
+ gpointer *vtable_slot;
+
+ g_assert (slot != -1);
+ vtable_offset = interface_offset + slot;
+ vtable_slot = & (vt->vtable [vtable_offset]);
#if DEBUG_IMT
printf ("mono_convert_imt_slot_to_vtable_slot: slot %p[%d] is in the IMT, and colliding becomes %p[%d] (interface_offset = %d, method->slot = %d)\n", slot, imt_slot, vtable_slot, vtable_offset, interface_offset, imt_method->slot);
#endif
#endif
/**
- * mono_magic_trampoline:
+ * common_call_trampoline:
*
- * This trampoline handles calls from JITted code.
+ * The code to handle normal, virtual, and interface method calls and jumps, both
+ * from JITted and LLVM compiled code.
*/
-gpointer
-mono_magic_trampoline (gssize *regs, guint8 *code, MonoMethod *m, guint8* tramp)
+static gpointer
+common_call_trampoline (mgreg_t *regs, guint8 *code, gpointer arg, guint8* tramp, MonoVTable *vt, gpointer *vtable_slot, gboolean need_rgctx_tramp)
{
- gpointer addr;
- gpointer *vtable_slot;
+ gpointer addr, compiled_method;
gboolean generic_shared = FALSE;
+ MonoMethod *m;
MonoMethod *declaring = NULL;
- MonoMethod *generic_virtual = NULL;
+ MonoMethod *generic_virtual = NULL, *variant_iface = NULL;
int context_used;
- gboolean proxy = FALSE;
+ gboolean proxy = FALSE, variance_used = FALSE;
+ gpointer *orig_vtable_slot;
+ MonoJitInfo *ji = NULL;
+
+ m = arg;
+
+ orig_vtable_slot = vtable_slot;
-#if MONO_ARCH_COMMON_VTABLE_TRAMPOLINE
if (m == MONO_FAKE_VTABLE_METHOD) {
int displacement;
- MonoVTable *vt = mono_arch_get_vcall_slot (code, (gpointer*)regs, &displacement);
if (!vt) {
int i;
MonoJitInfo *ji;
printf ("\n");
g_assert (vt);
}
+ displacement = (guint8*)vtable_slot - (guint8*)vt;
if (displacement > 0) {
- displacement -= G_STRUCT_OFFSET (MonoVTable, vtable);
- g_assert (displacement >= 0);
- displacement /= sizeof (gpointer);
+ int slot = (displacement - G_STRUCT_OFFSET (MonoVTable, vtable)) / sizeof (gpointer);
+ g_assert (slot >= 0);
/* Avoid loading metadata or creating a generic vtable if possible */
- addr = mono_aot_get_method_from_vt_slot (mono_domain_get (), vt, displacement);
+ addr = mono_aot_get_method_from_vt_slot (mono_domain_get (), vt, slot);
+ if (addr)
+ addr = mono_create_ftnptr (mono_domain_get (), addr);
if (addr && !vt->klass->valuetype) {
- vtable_slot = mono_arch_get_vcall_slot_addr (code, (gpointer*)regs);
+ vtable_slot = mono_get_vcall_slot_addr (code, regs);
if (mono_aot_is_got_entry (code, (guint8*)vtable_slot) || mono_domain_owns_vtable_slot (mono_domain_get (), vtable_slot)) {
*vtable_slot = mono_get_addr_from_ftnptr (addr);
}
return addr;
}
- m = mono_class_get_vtable_entry (vt->klass, displacement);
+ m = mono_class_get_vtable_entry (vt->klass, slot);
+ if (mono_method_needs_static_rgctx_invoke (m, FALSE))
+ need_rgctx_tramp = TRUE;
+
/*g_print ("%s with disp %d: %s at %p\n", vt->klass->name, displacement, m->name, code);*/
} else {
/* We got here from an interface method: redirect to IMT handling */
/*g_print ("vtable with disp %d at %p\n", displacement, code);*/
}
}
-#endif
+
/* this is the IMT trampoline */
#ifdef MONO_ARCH_HAVE_IMT
if (m == MONO_FAKE_IMT_METHOD) {
MonoMethod *impl_method;
- MonoGenericSharingContext *gsctx;
MonoObject *this_arg;
/* we get the interface method because mono_convert_imt_slot_to_vtable_slot ()
* needs the signature to be able to find the this argument
*/
- m = mono_arch_find_imt_method ((gpointer*)regs, code);
- vtable_slot = mono_arch_get_vcall_slot_addr (code, (gpointer*)regs);
+ m = mono_arch_find_imt_method (regs, code);
+ vtable_slot = orig_vtable_slot;
g_assert (vtable_slot);
- gsctx = mono_get_generic_context_from_code (code);
- this_arg = mono_arch_find_this_argument ((gpointer*)regs, m, gsctx);
+ this_arg = mono_arch_get_this_arg_from_call (NULL, mono_method_signature (m), regs, code);
if (this_arg->vtable->klass == mono_defaults.transparent_proxy_class) {
/* Use the slow path for now */
proxy = TRUE;
m = mono_object_get_virtual_method (this_arg, m);
} else {
- vtable_slot = mono_convert_imt_slot_to_vtable_slot (vtable_slot, (gpointer*)regs, code, m, &impl_method);
+ vtable_slot = mono_convert_imt_slot_to_vtable_slot (vtable_slot, regs, code, m, &impl_method, &need_rgctx_tramp, &variance_used);
/* mono_convert_imt_slot_to_vtable_slot () also gives us the method that is supposed
* to be called, so we compile it and go ahead as usual.
*/
if (m->is_inflated && ((MonoMethodInflated*)m)->context.method_inst) {
/* Generic virtual method */
generic_virtual = m;
- m = impl_method;
- m = mono_marshal_get_static_rgctx_invoke (m);
- } else {
- m = impl_method;
+ need_rgctx_tramp = TRUE;
+ } else if (variance_used && mono_class_has_variant_generic_params (m->klass)) {
+ variant_iface = m;
}
+ m = impl_method;
+ }
+ }
+#endif
+
+#ifdef MONO_ARCH_LLVM_SUPPORTED
+ if (!vtable_slot && code && !need_rgctx_tramp && mono_method_needs_static_rgctx_invoke (m, FALSE)) {
+ /*
+ * Call this only if the called method is shared, cause it is slow/loads a lot of
+ * data in AOT.
+ */
+ ji = mini_jit_info_table_find (mono_domain_get (), (char*)code, NULL);
+ if (ji && ji->from_llvm) {
+ /* LLVM can't pass an rgctx arg */
+ need_rgctx_tramp = TRUE;
}
}
#endif
g_assert (!m->klass->generic_container);
#ifdef MONO_ARCH_HAVE_IMT
- generic_virtual = mono_arch_find_imt_method ((gpointer*)regs, code);
+ generic_virtual = mono_arch_find_imt_method (regs, code);
#endif
if (generic_virtual) {
g_assert (generic_virtual->is_inflated);
m = mono_class_inflate_generic_method (declaring, &context);
/* FIXME: only do this if the method is sharable */
- m = mono_marshal_get_static_rgctx_invoke (m);
+ need_rgctx_tramp = TRUE;
} else if ((context_used = mono_method_check_context_used (m))) {
MonoClass *klass = NULL;
MonoMethod *actual_method = NULL;
if (m->is_inflated && mono_method_get_context (m)->method_inst) {
#ifdef MONO_ARCH_RGCTX_REG
- MonoMethodRuntimeGenericContext *mrgctx = (MonoMethodRuntimeGenericContext*)mono_arch_find_static_call_vtable ((gpointer*)regs, code);
+ MonoMethodRuntimeGenericContext *mrgctx = (MonoMethodRuntimeGenericContext*)mono_arch_find_static_call_vtable (regs, code);
klass = mrgctx->class_vtable->klass;
method_inst = mrgctx->method_inst;
#endif
} else if ((m->flags & METHOD_ATTRIBUTE_STATIC) || m->klass->valuetype) {
#ifdef MONO_ARCH_RGCTX_REG
- MonoVTable *vtable = mono_arch_find_static_call_vtable ((gpointer*)regs, code);
+ MonoVTable *vtable = mono_arch_find_static_call_vtable (regs, code);
klass = vtable->klass;
#else
#endif
} else {
#ifdef MONO_ARCH_HAVE_IMT
- MonoObject *this_argument = mono_arch_find_this_argument ((gpointer*)regs, m,
- mono_get_generic_context_from_code (code));
+ MonoObject *this_argument = mono_arch_get_this_arg_from_call (NULL, mono_method_signature (m), regs, code);
vt = this_argument->vtable;
- vtable_slot = mono_arch_get_vcall_slot_addr (code, (gpointer*)regs);
+ vtable_slot = orig_vtable_slot;
g_assert (this_argument->vtable->klass->inited);
//mono_class_init (this_argument->vtable->klass);
}
g_assert (klass);
+ g_assert (actual_method);
g_assert (actual_method->klass == klass);
if (actual_method->is_inflated)
MonoJitInfo *ji;
if (code)
- ji = mono_jit_info_table_find (mono_domain_get (), (char*)code);
+ ji = mini_jit_info_table_find (mono_domain_get (), (char*)code, NULL);
else
ji = NULL;
/* Calls made through delegates on platforms without delegate trampolines */
if (!code && mono_method_needs_static_rgctx_invoke (m, FALSE))
- m = mono_marshal_get_static_rgctx_invoke (m);
+ need_rgctx_tramp = TRUE;
- addr = mono_compile_method (m);
+ addr = compiled_method = mono_compile_method (m);
g_assert (addr);
- mono_debugger_trampoline_compiled (m, addr);
+ mono_debugger_trampoline_compiled (code, m, addr);
- if (generic_virtual) {
- int displacement;
- MonoVTable *vt = mono_arch_get_vcall_slot (code, (gpointer*)regs, &displacement);
+ if (need_rgctx_tramp)
+ addr = mono_create_static_rgctx_trampoline (m, addr);
+
+ if (generic_virtual || variant_iface) {
+ MonoMethod *target = generic_virtual ? generic_virtual : variant_iface;
- vtable_slot = mono_arch_get_vcall_slot_addr (code, (gpointer*)regs);
+ vtable_slot = orig_vtable_slot;
g_assert (vtable_slot);
- if (vt->klass->valuetype)
- addr = get_unbox_trampoline (mono_get_generic_context_from_code (code), m, addr);
+ if (vt->klass->valuetype) /*FIXME is this required variant iface?*/
+ addr = get_unbox_trampoline (mono_get_generic_context_from_code (code), m, addr, need_rgctx_tramp);
mono_method_add_generic_virtual_invocation (mono_domain_get (),
vt, vtable_slot,
- generic_virtual, addr);
+ target, addr);
return addr;
}
return addr;
}
- vtable_slot = mono_arch_get_vcall_slot_addr (code, (gpointer*)regs);
+ vtable_slot = orig_vtable_slot;
if (vtable_slot) {
+ gboolean variance_used = FALSE;
if (m->klass->valuetype)
- addr = get_unbox_trampoline (mono_get_generic_context_from_code (code), m, addr);
-
+ addr = get_unbox_trampoline (mono_get_generic_context_from_code (code), m, addr, need_rgctx_tramp);
g_assert (*vtable_slot);
if (!proxy && (mono_aot_is_got_entry (code, (guint8*)vtable_slot) || mono_domain_owns_vtable_slot (mono_domain_get (), vtable_slot))) {
#ifdef MONO_ARCH_HAVE_IMT
- vtable_slot = mono_convert_imt_slot_to_vtable_slot (vtable_slot, (gpointer*)regs, code, m, NULL);
+ vtable_slot = mono_convert_imt_slot_to_vtable_slot (vtable_slot, regs, code, m, NULL, &need_rgctx_tramp, &variance_used);
#endif
*vtable_slot = mono_get_addr_from_ftnptr (addr);
}
guint8 *plt_entry = mono_aot_get_plt_entry (code);
if (plt_entry) {
- mono_arch_patch_plt_entry (plt_entry, addr);
- } else if (!generic_shared || (m->iflags & METHOD_IMPL_ATTRIBUTE_INTERNAL_CALL) ||
- mono_domain_lookup_shared_generic (mono_domain_get (), declaring)) {
+ mono_aot_patch_plt_entry (plt_entry, NULL, regs, addr);
+ } else {
if (generic_shared) {
if (m->wrapper_type != MONO_WRAPPER_NONE)
m = mono_marshal_method_from_wrapper (m);
}
/* Patch calling code */
- if (plt_entry) {
-
- } else {
- MonoJitInfo *ji =
- mono_jit_info_table_find (mono_domain_get (), (char*)code);
+ {
MonoJitInfo *target_ji =
- mono_jit_info_table_find (mono_domain_get (), mono_get_addr_from_ftnptr (addr));
+ mono_jit_info_table_find (mono_domain_get (), mono_get_addr_from_ftnptr (compiled_method));
+
+ if (!ji)
+ ji = mono_jit_info_table_find (mono_domain_get (), (char*)code);
if (mono_method_same_domain (ji, target_ji))
mono_arch_patch_callsite (ji->code_start, code, addr);
return addr;
}
+/**
+ * mono_magic_trampoline:
+ *
+ * This trampoline handles calls from JITted code.
+ */
gpointer
-mono_generic_virtual_remoting_trampoline (gssize *regs, guint8 *code, MonoMethod *m, guint8 *tramp)
+mono_magic_trampoline (mgreg_t *regs, guint8 *code, gpointer arg, guint8* tramp)
+{
+ gpointer *vtable_slot;
+ int displacement;
+ MonoVTable *vt;
+
+ trampoline_calls ++;
+
+ if (code && !mono_use_llvm)
+ vt = mono_arch_get_vcall_slot (code, regs, &displacement);
+ else
+ vt = NULL;
+ if (vt)
+ vtable_slot = (gpointer*)((char*)vt + displacement);
+ else
+ vtable_slot = NULL;
+
+ return common_call_trampoline (regs, code, arg, tramp, vt, vtable_slot, FALSE);
+}
+
+#ifdef MONO_ARCH_LLVM_SUPPORTED
+/*
+ * mono_llvm_vcall_trampoline:
+ *
+ * This trampoline handles virtual calls when using LLVM.
+ */
+static gpointer
+mono_llvm_vcall_trampoline (mgreg_t *regs, guint8 *code, MonoMethod *m, guint8 *tramp)
+{
+ MonoObject *this;
+ MonoVTable *vt;
+ gpointer *vtable_slot;
+ int slot;
+
+ trampoline_calls ++;
+
+ /*
+ * We have the method which is called, we need to obtain the vtable slot without
+ * disassembly which is impossible with LLVM.
+ * So we use the this argument.
+ */
+ this = mono_arch_get_this_arg_from_call (NULL, mono_method_signature (m), regs, code);
+ g_assert (this);
+
+ slot = mono_method_get_vtable_slot (m);
+
+ g_assert (slot != -1);
+
+ g_assert (this->vtable->klass->vtable [slot] == m);
+
+ vt = this->vtable;
+
+ g_assert (!m->is_generic);
+
+ vtable_slot = &(vt->vtable [slot]);
+
+ return common_call_trampoline (regs, code, m, tramp, vt, vtable_slot, mono_method_needs_static_rgctx_invoke (m, 0));
+}
+#endif
+
+gpointer
+mono_generic_virtual_remoting_trampoline (mgreg_t *regs, guint8 *code, MonoMethod *m, guint8 *tramp)
{
MonoGenericContext context = { NULL, NULL };
MonoMethod *imt_method, *declaring;
gpointer addr;
+ trampoline_calls ++;
+
g_assert (m->is_generic);
if (m->is_inflated)
g_assert (!m->klass->generic_container);
#ifdef MONO_ARCH_HAVE_IMT
- imt_method = mono_arch_find_imt_method ((gpointer*)regs, code);
+ imt_method = mono_arch_find_imt_method (regs, code);
if (imt_method->is_inflated)
context.method_inst = ((MonoMethodInflated*)imt_method)->context.method_inst;
#endif
addr = mono_compile_method (m);
g_assert (addr);
- mono_debugger_trampoline_compiled (m, addr);
+ mono_debugger_trampoline_compiled (NULL, m, addr);
return addr;
}
* This trampoline handles calls made from AOT code. We try to bypass the
* normal JIT compilation logic to avoid loading the metadata for the method.
*/
-#ifdef MONO_ARCH_HAVE_CREATE_TRAMPOLINE_FROM_TOKEN
+#ifdef MONO_ARCH_AOT_SUPPORTED
gpointer
-mono_aot_trampoline (gssize *regs, guint8 *code, guint8 *token_info,
+mono_aot_trampoline (mgreg_t *regs, guint8 *code, guint8 *token_info,
guint8* tramp)
{
MonoImage *image;
MonoMethod *method = NULL;
gpointer addr;
gpointer *vtable_slot;
- gboolean is_got_entry;
guint8 *plt_entry;
+ trampoline_calls ++;
+
image = *(gpointer*)(gpointer)token_info;
token_info += sizeof (gpointer);
token = *(guint32*)(gpointer)token_info;
return mono_magic_trampoline (regs, code, method, tramp);
}
- vtable_slot = mono_arch_get_vcall_slot_addr (code, (gpointer*)regs);
+ addr = mono_create_ftnptr (mono_domain_get (), addr);
+
+ vtable_slot = mono_get_vcall_slot_addr (code, regs);
g_assert (!vtable_slot);
/* This is a normal call through a PLT entry */
plt_entry = mono_aot_get_plt_entry (code);
g_assert (plt_entry);
- mono_arch_patch_plt_entry (plt_entry, addr);
-
- is_got_entry = FALSE;
-
- /*
- * Since AOT code is only used in the root domain,
- * mono_domain_get () != mono_get_root_domain () means the calling method
- * is AppDomain:InvokeInDomain, so this is the same check as in
- * mono_method_same_domain () but without loading the metadata for the method.
- */
- if ((is_got_entry && (mono_domain_get () == mono_get_root_domain ())) || mono_domain_owns_vtable_slot (mono_domain_get (), vtable_slot)) {
-#ifdef MONO_ARCH_HAVE_IMT
- if (!method)
- method = mono_get_method (image, token, NULL);
- vtable_slot = mono_convert_imt_slot_to_vtable_slot (vtable_slot, (gpointer*)regs, code, method, NULL);
-#endif
- *vtable_slot = addr;
- }
+ mono_aot_patch_plt_entry (plt_entry, NULL, regs, addr);
return addr;
}
* This trampoline handles calls made from AOT code through the PLT table.
*/
gpointer
-mono_aot_plt_trampoline (gssize *regs, guint8 *code, guint8 *aot_module,
+mono_aot_plt_trampoline (mgreg_t *regs, guint8 *code, guint8 *aot_module,
guint8* tramp)
{
guint32 plt_info_offset = mono_aot_get_plt_info_offset (regs, code);
+ gpointer res;
+
+ trampoline_calls ++;
+
+ res = mono_aot_plt_resolve (aot_module, plt_info_offset, code);
+ if (!res) {
+ if (mono_loader_get_last_error ())
+ mono_raise_exception (mono_loader_error_prepare_exception (mono_loader_get_last_error ()));
+ // FIXME: Error handling (how ?)
+ g_assert (res);
+ }
- return mono_aot_plt_resolve (aot_module, plt_info_offset, code);
+ return res;
}
#endif
* for the type, then patches the caller code so it is not called again.
*/
void
-mono_class_init_trampoline (gssize *regs, guint8 *code, MonoVTable *vtable, guint8 *tramp)
+mono_class_init_trampoline (mgreg_t *regs, guint8 *code, MonoVTable *vtable, guint8 *tramp)
{
guint8 *plt_entry = mono_aot_get_plt_entry (code);
+ trampoline_calls ++;
+
mono_runtime_class_init (vtable);
if (plt_entry) {
- mono_arch_nullify_plt_entry (plt_entry);
+ mono_arch_nullify_plt_entry (plt_entry, regs);
} else {
mono_arch_nullify_class_init_trampoline (code, regs);
}
* for the type.
*/
void
-mono_generic_class_init_trampoline (gssize *regs, guint8 *code, MonoVTable *vtable, guint8 *tramp)
+mono_generic_class_init_trampoline (mgreg_t *regs, guint8 *code, MonoVTable *vtable, guint8 *tramp)
{
- g_assert (!vtable->initialized);
+ trampoline_calls ++;
mono_runtime_class_init (vtable);
}
static gpointer
-mono_rgctx_lazy_fetch_trampoline (gssize *regs, guint8 *code, gpointer data, guint8 *tramp)
+mono_rgctx_lazy_fetch_trampoline (mgreg_t *regs, guint8 *code, gpointer data, guint8 *tramp)
{
#ifdef MONO_ARCH_VTABLE_REG
static gboolean inited = FALSE;
static int num_lookups = 0;
guint32 slot = GPOINTER_TO_UINT (data);
- gpointer arg = (gpointer)(gssize)regs [MONO_ARCH_VTABLE_REG];
+ mgreg_t *r = (mgreg_t*)regs;
+ gpointer arg = (gpointer)(gssize)r [MONO_ARCH_VTABLE_REG];
guint32 index = MONO_RGCTX_SLOT_INDEX (slot);
gboolean mrgctx = MONO_RGCTX_SLOT_IS_MRGCTX (slot);
+ trampoline_calls ++;
+
if (!inited) {
mono_counters_register ("RGCTX unmanaged lookups", MONO_COUNTER_GENERICS | MONO_COUNTER_INT, &num_lookups);
inited = TRUE;
}
void
-mono_monitor_enter_trampoline (gssize *regs, guint8 *code, MonoObject *obj, guint8 *tramp)
+mono_monitor_enter_trampoline (mgreg_t *regs, guint8 *code, MonoObject *obj, guint8 *tramp)
{
mono_monitor_enter (obj);
}
void
-mono_monitor_exit_trampoline (gssize *regs, guint8 *code, MonoObject *obj, guint8 *tramp)
+mono_monitor_exit_trampoline (mgreg_t *regs, guint8 *code, MonoObject *obj, guint8 *tramp)
{
mono_monitor_exit (obj);
}
* This is called once the first time a delegate is invoked, so it must be fast.
*/
gpointer
-mono_delegate_trampoline (gssize *regs, guint8 *code, gpointer *tramp_data, guint8* tramp)
+mono_delegate_trampoline (mgreg_t *regs, guint8 *code, gpointer *tramp_data, guint8* tramp)
{
MonoDomain *domain = mono_domain_get ();
MonoDelegate *delegate;
MonoJitInfo *ji;
MonoMethod *m;
MonoMethod *method = NULL;
- gboolean multicast, callvirt;
+ gboolean multicast, callvirt = FALSE;
+ gboolean need_rgctx_tramp = FALSE;
MonoMethod *invoke = tramp_data [0];
guint8 *impl_this = tramp_data [1];
guint8 *impl_nothis = tramp_data [2];
+ MonoError err;
+ MonoMethodSignature *sig;
+
+ trampoline_calls ++;
/* Obtain the delegate object according to the calling convention */
* (ctor_with_method () does this, but it doesn't store the wrapper back into
* delegate->method).
*/
- if (delegate->target && delegate->target->vtable->klass == mono_defaults.transparent_proxy_class)
- method = mono_marshal_get_remoting_invoke (method);
- else if (mono_method_signature (method)->hasthis && method->klass->valuetype)
- method = mono_marshal_get_unbox_wrapper (method);
+ if (delegate->target && delegate->target->vtable->klass == mono_defaults.transparent_proxy_class) {
+#ifndef DISABLE_COM
+ if (((MonoTransparentProxy *)delegate->target)->remote_class->proxy_class != mono_defaults.com_object_class &&
+ !((MonoTransparentProxy *)delegate->target)->remote_class->proxy_class->is_com_object)
+#endif
+ method = mono_marshal_get_remoting_invoke (method);
+ }
+ else {
+ mono_error_init (&err);
+ sig = mono_method_signature_checked (method, &err);
+ if (!sig)
+ mono_error_raise_exception (&err);
+
+ if (sig->hasthis && method->klass->valuetype)
+ method = mono_marshal_get_unbox_wrapper (method);
+ }
} else {
ji = mono_jit_info_table_find (domain, mono_get_addr_from_ftnptr (delegate->method_ptr));
if (ji)
method = ji->method;
}
- callvirt = !delegate->target && method && mono_method_signature (method)->hasthis;
+
+ if (method) {
+ mono_error_init (&err);
+ sig = mono_method_signature_checked (method, &err);
+ if (!sig)
+ mono_error_raise_exception (&err);
+
+ callvirt = !delegate->target && sig->hasthis;
+ }
if (method && method->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED)
method = mono_marshal_get_synchronized_wrapper (method);
if (method && mono_method_needs_static_rgctx_invoke (method, FALSE))
- method = mono_marshal_get_static_rgctx_invoke (method);
+ need_rgctx_tramp = TRUE;
/*
* If the called address is a trampoline, replace it with the compiled method so
delegate->method_ptr = *delegate->method_code;
} else {
delegate->method_ptr = mono_compile_method (method);
+ if (need_rgctx_tramp)
+ delegate->method_ptr = mono_create_static_rgctx_trampoline (method, delegate->method_ptr);
if (delegate->method_code)
*delegate->method_code = delegate->method_ptr;
- mono_debugger_trampoline_compiled (method, delegate->method_ptr);
+ mono_debugger_trampoline_compiled (NULL, method, delegate->method_ptr);
}
+ } else {
+ if (need_rgctx_tramp)
+ delegate->method_ptr = mono_create_static_rgctx_trampoline (method, delegate->method_ptr);
}
multicast = ((MonoMulticastDelegate*)delegate)->prev != NULL;
m = mono_marshal_get_delegate_invoke (invoke, delegate);
code = mono_compile_method (m);
delegate->invoke_impl = mono_get_addr_from_ftnptr (code);
- mono_debugger_trampoline_compiled (m, delegate->invoke_impl);
+ mono_debugger_trampoline_compiled (NULL, m, delegate->invoke_impl);
return code;
}
#endif
+#ifdef MONO_ARCH_HAVE_HANDLER_BLOCK_GUARD
+static gpointer
+mono_handler_block_guard_trampoline (mgreg_t *regs, guint8 *code, gpointer *tramp_data, guint8* tramp)
+{
+ MonoContext ctx;
+ MonoException *exc;
+ MonoJitTlsData *jit_tls = TlsGetValue (mono_jit_tls_id);
+ gpointer resume_ip = jit_tls->handler_block_return_address;
+
+ memcpy (&ctx, &jit_tls->ex_ctx, sizeof (MonoContext));
+ MONO_CONTEXT_SET_IP (&ctx, jit_tls->handler_block_return_address);
+
+ jit_tls->handler_block_return_address = NULL;
+ jit_tls->handler_block = NULL;
+
+ if (!resume_ip) /*this should not happen, but we should avoid crashing */
+ exc = mono_get_exception_execution_engine ("Invalid internal state, resuming abort after handler block but no resume ip found");
+ else
+ exc = mono_thread_resume_interruption ();
+
+ if (exc) {
+ static void (*restore_context) (MonoContext *);
+
+ if (!restore_context)
+ restore_context = mono_get_restore_context ();
+
+ mono_handle_exception (&ctx, exc, NULL, FALSE);
+ restore_context (&ctx);
+ }
+
+ return resume_ip;
+}
+
+gpointer
+mono_create_handler_block_trampoline (void)
+{
+ static gpointer code;
+
+ if (mono_aot_only) {
+ g_assert (0);
+ return code;
+ }
+
+ mono_trampolines_lock ();
+
+ if (!code)
+ code = mono_arch_create_handler_block_trampoline ();
+
+ mono_trampolines_unlock ();
+
+ return code;
+}
+#endif
+
/*
* mono_get_trampoline_func:
*
return mono_monitor_enter_trampoline;
case MONO_TRAMPOLINE_MONITOR_EXIT:
return mono_monitor_exit_trampoline;
+#ifdef MONO_ARCH_LLVM_SUPPORTED
+ case MONO_TRAMPOLINE_LLVM_VCALL:
+ return mono_llvm_vcall_trampoline;
+#endif
+#ifdef MONO_ARCH_HAVE_HANDLER_BLOCK_GUARD
+ case MONO_TRAMPOLINE_HANDLER_BLOCK_GUARD:
+ return mono_handler_block_guard_trampoline;
+#endif
default:
g_assert_not_reached ();
return NULL;
}
}
+static guchar*
+create_trampoline_code (MonoTrampolineType tramp_type)
+{
+ MonoTrampInfo *info;
+ guchar *code;
+
+ code = mono_arch_create_generic_trampoline (tramp_type, &info, FALSE);
+ if (info) {
+ mono_save_trampoline_xdebug_info (info->name, info->code, info->code_size, info->unwind_ops);
+ mono_tramp_info_free (info);
+ }
+
+ return code;
+}
+
void
mono_trampolines_init (void)
{
if (mono_aot_only)
return;
- mono_trampoline_code [MONO_TRAMPOLINE_JIT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_JIT);
- mono_trampoline_code [MONO_TRAMPOLINE_JUMP] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_JUMP);
- mono_trampoline_code [MONO_TRAMPOLINE_CLASS_INIT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_CLASS_INIT);
- mono_trampoline_code [MONO_TRAMPOLINE_GENERIC_CLASS_INIT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_GENERIC_CLASS_INIT);
- mono_trampoline_code [MONO_TRAMPOLINE_RGCTX_LAZY_FETCH] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_RGCTX_LAZY_FETCH);
+ mono_trampoline_code [MONO_TRAMPOLINE_JIT] = create_trampoline_code (MONO_TRAMPOLINE_JIT);
+ mono_trampoline_code [MONO_TRAMPOLINE_JUMP] = create_trampoline_code (MONO_TRAMPOLINE_JUMP);
+ mono_trampoline_code [MONO_TRAMPOLINE_CLASS_INIT] = create_trampoline_code (MONO_TRAMPOLINE_CLASS_INIT);
+ mono_trampoline_code [MONO_TRAMPOLINE_GENERIC_CLASS_INIT] = create_trampoline_code (MONO_TRAMPOLINE_GENERIC_CLASS_INIT);
+ mono_trampoline_code [MONO_TRAMPOLINE_RGCTX_LAZY_FETCH] = create_trampoline_code (MONO_TRAMPOLINE_RGCTX_LAZY_FETCH);
#ifdef MONO_ARCH_AOT_SUPPORTED
- mono_trampoline_code [MONO_TRAMPOLINE_AOT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_AOT);
- mono_trampoline_code [MONO_TRAMPOLINE_AOT_PLT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_AOT_PLT);
+ mono_trampoline_code [MONO_TRAMPOLINE_AOT] = create_trampoline_code (MONO_TRAMPOLINE_AOT);
+ mono_trampoline_code [MONO_TRAMPOLINE_AOT_PLT] = create_trampoline_code (MONO_TRAMPOLINE_AOT_PLT);
#endif
#ifdef MONO_ARCH_HAVE_CREATE_DELEGATE_TRAMPOLINE
- mono_trampoline_code [MONO_TRAMPOLINE_DELEGATE] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_DELEGATE);
+ mono_trampoline_code [MONO_TRAMPOLINE_DELEGATE] = create_trampoline_code (MONO_TRAMPOLINE_DELEGATE);
+#endif
+ mono_trampoline_code [MONO_TRAMPOLINE_RESTORE_STACK_PROT] = create_trampoline_code (MONO_TRAMPOLINE_RESTORE_STACK_PROT);
+ mono_trampoline_code [MONO_TRAMPOLINE_GENERIC_VIRTUAL_REMOTING] = create_trampoline_code (MONO_TRAMPOLINE_GENERIC_VIRTUAL_REMOTING);
+ mono_trampoline_code [MONO_TRAMPOLINE_MONITOR_ENTER] = create_trampoline_code (MONO_TRAMPOLINE_MONITOR_ENTER);
+ mono_trampoline_code [MONO_TRAMPOLINE_MONITOR_EXIT] = create_trampoline_code (MONO_TRAMPOLINE_MONITOR_EXIT);
+#ifdef MONO_ARCH_LLVM_SUPPORTED
+ mono_trampoline_code [MONO_TRAMPOLINE_LLVM_VCALL] = create_trampoline_code (MONO_TRAMPOLINE_LLVM_VCALL);
#endif
- mono_trampoline_code [MONO_TRAMPOLINE_RESTORE_STACK_PROT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_RESTORE_STACK_PROT);
- mono_trampoline_code [MONO_TRAMPOLINE_GENERIC_VIRTUAL_REMOTING] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_GENERIC_VIRTUAL_REMOTING);
- mono_trampoline_code [MONO_TRAMPOLINE_MONITOR_ENTER] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_MONITOR_ENTER);
- mono_trampoline_code [MONO_TRAMPOLINE_MONITOR_EXIT] = mono_arch_create_trampoline_code (MONO_TRAMPOLINE_MONITOR_EXIT);
+#ifdef MONO_ARCH_HAVE_HANDLER_BLOCK_GUARD
+ mono_trampoline_code [MONO_TRAMPOLINE_HANDLER_BLOCK_GUARD] = create_trampoline_code (MONO_TRAMPOLINE_HANDLER_BLOCK_GUARD);
+ mono_create_handler_block_trampoline ();
+#endif
+
+ mono_counters_register ("Calls to trampolines", MONO_COUNTER_JIT | MONO_COUNTER_INT, &trampoline_calls);
}
void
{
if (class_init_hash_addr)
g_hash_table_destroy (class_init_hash_addr);
- if (delegate_trampoline_hash_addr)
- g_hash_table_destroy (delegate_trampoline_hash_addr);
DeleteCriticalSection (&trampolines_mutex);
}
if (!code) {
if (mono_aot_only)
- code = mono_aot_get_named_code ("generic_class_init_trampoline");
+ /* get_named_code () might return an ftnptr, but our caller expects a direct pointer */
+ code = mono_get_addr_from_ftnptr (mono_aot_get_trampoline ("generic_class_init_trampoline"));
else
- code = mono_arch_create_generic_class_init_trampoline ();
+ code = mono_arch_create_generic_class_init_trampoline (NULL, FALSE);
}
mono_trampolines_unlock ();
code = mono_create_specific_trampoline (method, MONO_TRAMPOLINE_JUMP, mono_domain_get (), &code_size);
g_assert (code_size);
- ji = mono_domain_alloc0 (domain, sizeof (MonoJitInfo));
+ ji = mono_domain_alloc0 (domain, MONO_SIZEOF_JIT_INFO);
ji->code_start = code;
ji->code_size = code_size;
ji->method = method;
return mono_create_jit_trampoline_in_domain (mono_domain_get (), method);
}
-#ifdef MONO_ARCH_HAVE_CREATE_TRAMPOLINE_FROM_TOKEN
gpointer
mono_create_jit_trampoline_from_token (MonoImage *image, guint32 token)
{
return tramp;
}
-#endif
gpointer
mono_create_delegate_trampoline (MonoClass *klass)
tramp_data = mono_domain_alloc (domain, sizeof (gpointer) * 3);
tramp_data [0] = invoke;
- if (mono_aot_only) {
- tramp_data [1] = NULL;
- tramp_data [2] = NULL;
- } else {
- tramp_data [1] = mono_arch_get_delegate_invoke_impl (mono_method_signature (invoke), TRUE);
- tramp_data [2] = mono_arch_get_delegate_invoke_impl (mono_method_signature (invoke), FALSE);
- }
+ tramp_data [1] = mono_arch_get_delegate_invoke_impl (mono_method_signature (invoke), TRUE);
+ tramp_data [2] = mono_arch_get_delegate_invoke_impl (mono_method_signature (invoke), FALSE);
ptr = mono_create_specific_trampoline (tramp_data, MONO_TRAMPOLINE_DELEGATE, mono_domain_get (), &code_size);
g_assert (code_size);
klass, ptr);
mono_domain_unlock (domain);
- mono_trampolines_lock ();
- if (!delegate_trampoline_hash_addr)
- delegate_trampoline_hash_addr = g_hash_table_new (NULL, NULL);
- g_hash_table_insert (delegate_trampoline_hash_addr, ptr, klass);
- mono_trampolines_unlock ();
-
return ptr;
#else
return NULL;
if (tramp)
return tramp;
- tramp = mono_arch_create_rgctx_lazy_fetch_trampoline (offset);
+ tramp = mono_arch_create_rgctx_lazy_fetch_trampoline (offset, NULL, FALSE);
ptr = mono_create_ftnptr (mono_get_root_domain (), tramp);
mono_trampolines_lock ();
if (mono_aot_only) {
if (!code)
- code = mono_aot_get_named_code ("monitor_enter_trampoline");
+ code = mono_aot_get_trampoline ("monitor_enter_trampoline");
return code;
}
mono_trampolines_lock ();
if (!code)
- code = mono_arch_create_monitor_enter_trampoline ();
+ code = mono_arch_create_monitor_enter_trampoline (NULL, FALSE);
mono_trampolines_unlock ();
#else
if (mono_aot_only) {
if (!code)
- code = mono_aot_get_named_code ("monitor_exit_trampoline");
+ code = mono_aot_get_trampoline ("monitor_exit_trampoline");
return code;
}
mono_trampolines_lock ();
if (!code)
- code = mono_arch_create_monitor_exit_trampoline ();
+ code = mono_arch_create_monitor_exit_trampoline (NULL, FALSE);
mono_trampolines_unlock ();
#else
#endif
return code;
}
+
+#ifdef MONO_ARCH_LLVM_SUPPORTED
+/*
+ * mono_create_llvm_vcall_trampoline:
+ *
+ * LLVM emits code for virtual calls which mono_get_vcall_slot is unable to
+ * decode, i.e. only the final branch address is available:
+ * mov <offset>(%rax), %rax
+ * <random code inserted by instruction scheduling>
+ * call *%rax
+ *
+ * To work around this problem, we don't use the common vtable trampoline when
+ * llvm is enabled. Instead, we use one trampoline per method.
+ */
+gpointer
+mono_create_llvm_vcall_trampoline (MonoMethod *method)
+{
+ MonoDomain *domain;
+ gpointer res;
+
+ domain = mono_domain_get ();
+
+ mono_domain_lock (domain);
+ res = g_hash_table_lookup (domain_jit_info (domain)->llvm_vcall_trampoline_hash, method);
+ mono_domain_unlock (domain);
+ if (res)
+ return res;
+
+ res = mono_create_specific_trampoline (method, MONO_TRAMPOLINE_LLVM_VCALL, domain, NULL);
+
+ mono_domain_lock (domain);
+ g_hash_table_insert (domain_jit_info (domain)->llvm_vcall_trampoline_hash, method, res);
+ mono_domain_unlock (domain);
+
+ return res;
+}
+
+/*
+ * mono_create_llvm_imt_trampoline:
+ *
+ * LLVM compiled code can't pass in the IMT argument, so we use this trampoline, which
+ * sets the IMT argument, then branches to the contents of the vtable slot given by
+ * vt_offset in the vtable which is obtained from the argument list.
+ */
+gpointer
+mono_create_llvm_imt_trampoline (MonoDomain *domain, MonoMethod *m, int vt_offset)
+{
+#ifdef MONO_ARCH_HAVE_LLVM_IMT_TRAMPOLINE
+ return mono_arch_get_llvm_imt_trampoline (domain, m, vt_offset);
+#else
+ g_assert_not_reached ();
+ return NULL;
+#endif
+}
+#endif
MonoVTable*
mono_find_class_init_trampoline_by_addr (gconstpointer addr)
return res;
}
-MonoClass*
-mono_find_delegate_trampoline_by_addr (gconstpointer addr)
-{
- MonoClass *res;
-
- mono_trampolines_lock ();
- if (delegate_trampoline_hash_addr)
- res = g_hash_table_lookup (delegate_trampoline_hash_addr, addr);
- else
- res = NULL;
- mono_trampolines_unlock ();
- return res;
-}
-
guint32
mono_find_rgctx_lazy_fetch_trampoline_by_addr (gconstpointer addr)
{
projects / mono.git / blobdiff