[JIT] Last two optimizations to Roslyn.
}
void
-mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
+mono_gc_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
{
g_assert_not_reached ();
}
+void*
+mono_gc_get_range_copy_func (void)
+{
+ return &mono_gc_wbarrier_range_copy;
+}
guint8*
mono_gc_get_card_table (int *shift_bits, gpointer *card_mask)
MonoMethod* mono_gc_get_write_barrier (void);
/* Fast valuetype copy */
-void mono_gc_wbarrier_value_copy_bitmap (gpointer dest, gpointer src, int size, unsigned bitmap);
+/* WARNING: [dest, dest + size] must be within the bounds of a single type, otherwise the GC will lose remset entries */
+void mono_gc_wbarrier_range_copy (gpointer dest, gpointer src, int size);
+void* mono_gc_get_range_copy_func (void);
+
/* helper for the managed alloc support */
MonoString *
STELEMREF_OBJECT, /*no check at all*/
STELEMREF_SEALED_CLASS, /*check vtable->klass->element_type */
STELEMREF_CLASS, /*only the klass->parents check*/
+ STELEMREF_CLASS_SMALL_IDEPTH, /* like STELEMREF_CLASS bit without the idepth check */
STELEMREF_INTERFACE, /*interfaces without variant generic arguments. */
STELEMREF_COMPLEX, /*arrays, MBR or types with variant generic args - go straight to icalls*/
STELEMREF_KIND_COUNT
};
static const char *strelemref_wrapper_name[] = {
- "object", "sealed_class", "class", "interface", "complex"
+ "object", "sealed_class", "class", "class_small_idepth", "interface", "complex"
};
static gboolean
return STELEMREF_COMPLEX;
if (mono_class_is_sealed (element_class))
return STELEMREF_SEALED_CLASS;
+ if (element_class->idepth <= MONO_DEFAULT_SUPERTABLE_SIZE)
+ return STELEMREF_CLASS_SMALL_IDEPTH;
+
return STELEMREF_CLASS;
}
mono_mb_emit_exception (mb, "ArrayTypeMismatchException", NULL);
break;
}
+
+ case STELEMREF_CLASS_SMALL_IDEPTH:
+ /*
+ the method:
+ <ldelema (bound check)>
+ if (!value)
+ goto do_store;
+
+ aklass = array->vtable->klass->element_class;
+ vklass = value->vtable->klass;
+
+ if (vklass->supertypes [aklass->idepth - 1] != aklass)
+ goto do_exception;
+
+ do_store:
+ *array_slot_addr = value;
+ return;
+
+ long:
+ throw new ArrayTypeMismatchException ();
+ */
+ aklass = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ vklass = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ array_slot_addr = mono_mb_add_local (mb, &mono_defaults.object_class->this_arg);
+
+ /* ldelema (implicit bound check) */
+ load_array_element_address (mb);
+ mono_mb_emit_stloc (mb, array_slot_addr);
+
+ /* if (!value) goto do_store */
+ mono_mb_emit_ldarg (mb, 2);
+ b1 = mono_mb_emit_branch (mb, CEE_BRFALSE);
+
+ /* aklass = array->vtable->klass->element_class */
+ load_array_class (mb, aklass);
+
+ /* vklass = value->vtable->klass */
+ load_value_class (mb, vklass);
+
+ /* if (vklass->supertypes [aklass->idepth - 1] != aklass) goto failure */
+ mono_mb_emit_ldloc (mb, vklass);
+ mono_mb_emit_ldflda (mb, MONO_STRUCT_OFFSET (MonoClass, supertypes));
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+
+ mono_mb_emit_ldloc (mb, aklass);
+ mono_mb_emit_ldflda (mb, MONO_STRUCT_OFFSET (MonoClass, idepth));
+ mono_mb_emit_byte (mb, CEE_LDIND_U2);
+ mono_mb_emit_icon (mb, 1);
+ mono_mb_emit_byte (mb, CEE_SUB);
+ mono_mb_emit_icon (mb, sizeof (void*));
+ mono_mb_emit_byte (mb, CEE_MUL);
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+
+ mono_mb_emit_ldloc (mb, aklass);
+ b4 = mono_mb_emit_branch (mb, CEE_BNE_UN);
+
+ /* do_store: */
+ mono_mb_patch_branch (mb, b1);
+ mono_mb_emit_ldloc (mb, array_slot_addr);
+ mono_mb_emit_ldarg (mb, 2);
+ mono_mb_emit_byte (mb, CEE_STIND_REF);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ /* do_exception: */
+ mono_mb_patch_branch (mb, b4);
+
+ mono_mb_emit_exception (mb, "ArrayTypeMismatchException", NULL);
+ break;
+
case STELEMREF_INTERFACE:
/*Mono *klass;
MonoVTable *vt;
}
void
-mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
+mono_gc_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
{
g_assert_not_reached ();
}
+void*
+mono_gc_get_range_copy_func (void)
+{
+ return &mono_gc_wbarrier_range_copy;
+}
+
guint8*
mono_gc_get_card_table (int *shift_bits, gpointer *card_mask)
{
}
void
-mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
+mono_gc_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
{
- sgen_wbarrier_value_copy_bitmap (_dest, _src, size, bitmap);
+ sgen_wbarrier_range_copy (_dest, _src, size);
+}
+
+void*
+mono_gc_get_range_copy_func (void)
+{
+ return sgen_get_remset ()->wbarrier_range_copy;
}
int
if (align < SIZEOF_VOID_P)
return FALSE;
- /*This value cannot be bigger than 32 due to the way we calculate the required wb bitmap.*/
- if (size > 32 * SIZEOF_VOID_P)
+ if (size > 5 * SIZEOF_VOID_P)
return FALSE;
create_write_barrier_bitmap (cfg, klass, &need_wb, 0);
- /* We don't unroll more than 5 stores to avoid code bloat. */
- if (size > 5 * SIZEOF_VOID_P) {
- /*This is harmless and simplify mono_gc_wbarrier_value_copy_bitmap */
- size += (SIZEOF_VOID_P - 1);
- size &= ~(SIZEOF_VOID_P - 1);
-
- EMIT_NEW_ICONST (cfg, iargs [2], size);
- EMIT_NEW_ICONST (cfg, iargs [3], need_wb);
- mono_emit_jit_icall (cfg, mono_gc_wbarrier_value_copy_bitmap, iargs);
- return TRUE;
- }
-
destreg = iargs [0]->dreg;
srcreg = iargs [1]->dreg;
offset = 0;
else
n = mono_class_value_size (klass, &align);
+ if (!align)
+ align = SIZEOF_VOID_P;
/* if native is true there should be no references in the struct */
if (cfg->gen_write_barriers && (klass->has_references || size_ins) && !native) {
/* Avoid barriers when storing to the stack */
/* It's ok to intrinsify under gsharing since shared code types are layout stable. */
if (!size_ins && (cfg->opt & MONO_OPT_INTRINS) && mono_emit_wb_aware_memcpy (cfg, klass, iargs, n, align)) {
return;
- } else if (context_used) {
- iargs [2] = mini_emit_get_rgctx_klass (cfg, context_used, klass, MONO_RGCTX_INFO_KLASS);
- } else {
- iargs [2] = emit_runtime_constant (cfg, MONO_PATCH_INFO_CLASS, klass);
- if (!cfg->compile_aot)
- mono_class_compute_gc_descriptor (klass);
- }
+ } else if (size_ins || align < SIZEOF_VOID_P) {
+ if (context_used) {
+ iargs [2] = mini_emit_get_rgctx_klass (cfg, context_used, klass, MONO_RGCTX_INFO_KLASS);
+ } else {
+ iargs [2] = emit_runtime_constant (cfg, MONO_PATCH_INFO_CLASS, klass);
+ if (!cfg->compile_aot)
+ mono_class_compute_gc_descriptor (klass);
+ }
+ if (size_ins)
+ mono_emit_jit_icall (cfg, mono_gsharedvt_value_copy, iargs);
+ else
+ mono_emit_jit_icall (cfg, mono_value_copy, iargs);
+ } else {
+ /* We don't unroll more than 5 stores to avoid code bloat. */
+ /*This is harmless and simplify mono_gc_get_range_copy_func */
+ n += (SIZEOF_VOID_P - 1);
+ n &= ~(SIZEOF_VOID_P - 1);
- if (size_ins)
- mono_emit_jit_icall (cfg, mono_gsharedvt_value_copy, iargs);
- else
- mono_emit_jit_icall (cfg, mono_value_copy, iargs);
- return;
+ EMIT_NEW_ICONST (cfg, iargs [2], n);
+ mono_emit_jit_icall (cfg, mono_gc_get_range_copy_func (), iargs);
+ }
}
}
register_icall (mono_gsharedvt_constrained_call, "mono_gsharedvt_constrained_call", "object ptr ptr ptr ptr ptr", FALSE);
register_icall (mono_gsharedvt_value_copy, "mono_gsharedvt_value_copy", "void ptr ptr ptr", TRUE);
- register_icall (mono_gc_wbarrier_value_copy_bitmap, "mono_gc_wbarrier_value_copy_bitmap", "void ptr ptr int int", FALSE);
+ register_icall_no_wrapper (mono_gc_get_range_copy_func (), "mono_gc_range_copy", "void ptr ptr int");
register_icall (mono_object_castclass_with_cache, "mono_object_castclass_with_cache", "object object ptr ptr", FALSE);
register_icall (mono_object_isinst_with_cache, "mono_object_isinst_with_cache", "object object ptr ptr", FALSE);
sgen_card_table_mark_address ((mword)ptr);
}
+static void
+sgen_card_table_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
+{
+ GCObject **dest = (GCObject **)_dest;
+ GCObject **src = (GCObject **)_src;
+
+ size_t nursery_bits = DEFAULT_NURSERY_BITS;
+ char *start = sgen_nursery_start;
+ G_GNUC_UNUSED char *end = sgen_nursery_end;
+
+ /*
+ * It's cardtable theory time!
+ * Our cardtable scanning code supports marking any card that an object/valuetype belongs to.
+ * This function is supposed to be used to copy a range that fully belongs to a single type.
+ * It must not be used, for example, to copy 2 adjacent VTs in an array.
+ */
+ volatile guint8 *card_address = (volatile guint8 *)sgen_card_table_get_card_address ((mword)dest);
+ while (size) {
+ GCObject *value = *src;
+ *dest = value;
+ if (SGEN_PTR_IN_NURSERY (value, nursery_bits, start, end) || concurrent_collection_in_progress) {
+ *card_address = 1;
+ sgen_dummy_use (value);
+ }
+ ++src;
+ ++dest;
+ size -= SIZEOF_VOID_P;
+ }
+}
+
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
guint8 *sgen_shadow_cardtable;
remset->find_address = sgen_card_table_find_address;
remset->find_address_with_cards = sgen_card_table_find_address_with_cards;
+ remset->wbarrier_range_copy = sgen_card_table_wbarrier_range_copy;
need_mod_union = sgen_get_major_collector ()->is_concurrent;
}
MonoCoopMutex sgen_interruption_mutex;
int current_collection_generation = -1;
-static volatile gboolean concurrent_collection_in_progress = FALSE;
+volatile gboolean concurrent_collection_in_progress = FALSE;
/* objects that are ready to be finalized */
static SgenPointerQueue fin_ready_queue = SGEN_POINTER_QUEUE_INIT (INTERNAL_MEM_FINALIZE_READY);
}
void
-sgen_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
+sgen_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
{
- GCObject **dest = (GCObject **)_dest;
- GCObject **src = (GCObject **)_src;
-
- while (size) {
- if (bitmap & 0x1)
- mono_gc_wbarrier_generic_store (dest, *src);
- else
- *dest = *src;
- ++src;
- ++dest;
- size -= SIZEOF_VOID_P;
- bitmap >>= 1;
- }
+ remset.wbarrier_range_copy (_dest,_src, size);
}
/*
void (*wbarrier_object_copy) (GCObject* obj, GCObject *src);
void (*wbarrier_generic_nostore) (gpointer ptr);
void (*record_pointer) (gpointer ptr);
+ void (*wbarrier_range_copy) (gpointer dest, gpointer src, int count);
void (*start_scan_remsets) (void);
void mono_gc_wbarrier_generic_store (gpointer ptr, GCObject* value);
void mono_gc_wbarrier_generic_store_atomic (gpointer ptr, GCObject *value);
-void sgen_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap);
+void sgen_wbarrier_range_copy (gpointer _dest, gpointer _src, int size);
static inline SgenDescriptor
sgen_obj_get_descriptor (GCObject *obj)
extern mword total_allocated_major;
extern volatile gboolean sgen_suspend_finalizers;
extern MonoCoopMutex gc_mutex;
+extern volatile gboolean concurrent_collection_in_progress;
/* Nursery helpers. */
projects / mono.git / commitdiff