*
* Author:
* Paolo Molaro (lupus@ximian.com)
+ * Rodrigo Kumpera (kumpera@gmail.com)
*
* Copyright 2005-2010 Novell, Inc (http://www.novell.com)
*
*
* Copyright 2001-2003 Ximian, Inc
* Copyright 2003-2010 Novell, Inc.
+ * Copyright 2011 Xamarin, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
A good place to start is add_nursery_frag. The tricky thing here is
placing those objects atomically outside of a collection.
-
+ *) Allocation should use asymmetric Dekker synchronization:
+ http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
+ This should help weak consistency archs.
*/
#include "config.h"
#ifdef HAVE_SGEN_GC
#include "utils/mono-semaphore.h"
#include "utils/mono-counters.h"
#include "utils/mono-proclib.h"
+#include "utils/mono-memory-model.h"
#include <mono/utils/memcheck.h>
int gc_debug_level = 0;
FILE* gc_debug_file;
+static gboolean debug_print_allowance = FALSE;
/*
void
#endif
#ifndef DISABLE_CRITICAL_REGION
-/* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
-#define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
-#define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
+
+/* Enter must be visible before anything is done in the critical region. */
+#define ENTER_CRITICAL_REGION do { mono_atomic_store_release (&IN_CRITICAL_REGION, 1); } while (0)
+
+/* Exit must make sure all critical regions stores are visible before it signal the end of the region.
+ * We don't need to emit a full barrier since we
+ */
+#define EXIT_CRITICAL_REGION do { mono_atomic_store_seq (&IN_CRITICAL_REGION, 0); } while (0)
+
+
#endif
/*
typedef SgenGrayQueue GrayQueue;
-typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
-
/* forward declarations */
static int stop_world (int generation);
static int restart_world (int generation);
return bitmap;
}
+ case DESC_TYPE_LARGE_BITMAP: {
+ gsize bmap = (d >> LOW_TYPE_BITS) << OBJECT_HEADER_WORDS;
+
+ bitmap = g_new0 (gsize, 1);
+ bitmap [0] = bmap;
+ *numbits = 0;
+ while (bmap) {
+ (*numbits) ++;
+ bmap >>= 1;
+ }
+ return bitmap;
+ }
default:
g_assert_not_reached ();
}
char *obj;
if (current_collection_generation == GENERATION_NURSERY) {
+ ScanObjectFunc scan_func = mono_sgen_get_minor_scan_object ();
+
for (;;) {
GRAY_OBJECT_DEQUEUE (queue, obj);
if (!obj)
return TRUE;
DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
- major_collector.minor_scan_object (obj, queue);
+ scan_func (obj, queue);
}
} else {
int i;
finalized_array [finalized_array_entries++] = object;
}
+static void
+bridge_process (void)
+{
+ if (finalized_array_entries <= 0)
+ return;
+
+ g_assert (mono_sgen_need_bridge_processing ());
+ mono_sgen_bridge_processing (finalized_array_entries, finalized_array);
+
+ finalized_array_entries = 0;
+}
+
CopyOrMarkObjectFunc
mono_sgen_get_copy_object (void)
{
}
}
+ScanObjectFunc
+mono_sgen_get_minor_scan_object (void)
+{
+ g_assert (current_collection_generation == GENERATION_NURSERY);
+
+ if (collection_is_parallel ())
+ return major_collector.minor_scan_object;
+ else
+ return major_collector.nopar_minor_scan_object;
+}
+
+ScanVTypeFunc
+mono_sgen_get_minor_scan_vtype (void)
+{
+ g_assert (current_collection_generation == GENERATION_NURSERY);
+
+ if (collection_is_parallel ())
+ return major_collector.minor_scan_vtype;
+ else
+ return major_collector.nopar_minor_scan_vtype;
+}
+
static void
finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
{
if (generation == GENERATION_OLD)
finalize_in_range (copy_func, nursery_start, nursery_end, GENERATION_NURSERY, queue);
- if (fin_ready != num_ready_finalizers) {
+ if (fin_ready != num_ready_finalizers)
++num_loops;
- if (finalized_array != NULL)
- mono_sgen_bridge_processing (finalized_array_entries, finalized_array);
- }
/* drain the new stack that might have been created */
DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * major_collector.section_size + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
+ if (debug_print_allowance) {
+ mword old_major = last_collection_old_num_major_sections * major_collector.section_size;
+ mword new_major = num_major_sections * major_collector.section_size;
+
+ fprintf (gc_debug_file, "Before collection: %ld bytes (%ld major, %ld LOS)\n",
+ old_major + last_collection_old_los_memory_usage, old_major, last_collection_old_los_memory_usage);
+ fprintf (gc_debug_file, "After collection: %ld bytes (%ld major, %ld LOS)\n",
+ new_major + last_collection_los_memory_usage, new_major, last_collection_los_memory_usage);
+ fprintf (gc_debug_file, "Allowance: %ld bytes\n", minor_collection_allowance);
+ }
+
if (major_collector.have_computed_minor_collection_allowance)
major_collector.have_computed_minor_collection_allowance ();
*/
static void*
-alloc_degraded (MonoVTable *vtable, size_t size)
-{
+alloc_degraded (MonoVTable *vtable, size_t size, gboolean for_mature)
+{
+ static int last_major_gc_warned = -1;
+ static int num_degraded = 0;
+
+ if (!for_mature) {
+ if (last_major_gc_warned < num_major_gcs) {
+ ++num_degraded;
+ if (num_degraded == 1 || num_degraded == 3)
+ fprintf (stderr, "Warning: Degraded allocation. Consider increasing nursery-size if the warning persists.\n");
+ else if (num_degraded == 10)
+ fprintf (stderr, "Warning: Repeated degraded allocation. Consider increasing nursery-size.\n");
+ last_major_gc_warned = num_major_gcs;
+ }
+ }
+
if (need_major_collection (0)) {
mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
stop_world (1);
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p , vtable, size);
g_assert (*p == NULL);
- *p = vtable;
+ mono_atomic_store_seq (p, vtable);
return p;
}
* for a while, to decrease the number of useless nursery collections.
*/
if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
- p = alloc_degraded (vtable, size);
+ p = alloc_degraded (vtable, size, FALSE);
binary_protocol_alloc_degraded (p, vtable, size);
return p;
}
if (!p) {
minor_collect_or_expand_inner (size);
if (degraded_mode) {
- p = alloc_degraded (vtable, size);
+ p = alloc_degraded (vtable, size, FALSE);
binary_protocol_alloc_degraded (p, vtable, size);
return p;
} else {
if (!p) {
minor_collect_or_expand_inner (tlab_size);
if (degraded_mode) {
- p = alloc_degraded (vtable, size);
+ p = alloc_degraded (vtable, size, FALSE);
binary_protocol_alloc_degraded (p, vtable, size);
return p;
} else {
if (G_LIKELY (p)) {
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p, vtable, size);
- *p = vtable;
+ mono_atomic_store_seq (p, vtable);
}
return p;
if (!p)
return NULL;
+ /*FIXME we should use weak memory ops here. Should help specially on x86. */
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
memset (p, 0, size);
} else {
}
}
- /*
- * FIXME: We might need a memory barrier here so the change to tlab_next is
- * visible before the vtable store.
- */
-
HEAVY_STAT (++stat_objects_alloced);
HEAVY_STAT (stat_bytes_alloced += size);
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p, vtable, size);
- g_assert (*p == NULL);
- *p = vtable;
+ g_assert (*p == NULL); /* FIXME disable this in non debug builds */
+
+ mono_atomic_store_seq (p, vtable);
return p;
}
ENTER_CRITICAL_REGION;
arr = mono_gc_try_alloc_obj_nolock (vtable, size);
if (arr) {
+ /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
arr->max_length = max_length;
EXIT_CRITICAL_REGION;
return arr;
ENTER_CRITICAL_REGION;
str = mono_gc_try_alloc_obj_nolock (vtable, size);
if (str) {
+ /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
str->length = len;
EXIT_CRITICAL_REGION;
return str;
if (G_LIKELY (p)) {
DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc_pinned (p, vtable, size);
- *p = vtable;
+ mono_atomic_store_seq (p, vtable);
}
UNLOCK_GC;
return p;
void **res;
size_t size = ALIGN_UP (vtable->klass->instance_size);
LOCK_GC;
- res = alloc_degraded (vtable, size);
- *res = vtable;
+ res = alloc_degraded (vtable, size, TRUE);
+ mono_atomic_store_seq (res, vtable);
UNLOCK_GC;
if (G_UNLIKELY (vtable->klass->has_finalize))
mono_object_register_finalizer ((MonoObject*)res);
max_pause_usec = MAX (usec, max_pause_usec);
DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
mono_profiler_gc_event (MONO_GC_EVENT_POST_START_WORLD, generation);
+
+ bridge_process ();
+
return count;
}
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
return p + 1;
- major_collector.minor_scan_object ((char*)ptr, queue);
+ mono_sgen_get_minor_scan_object () ((char*)ptr, queue);
return p + 1;
case REMSET_VTYPE: {
+ ScanVTypeFunc scan_vtype = mono_sgen_get_minor_scan_vtype ();
size_t skip_size;
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
count = p [2];
skip_size = p [3];
while (count-- > 0) {
- major_collector.minor_scan_vtype ((char*)ptr, desc, queue);
+ scan_vtype ((char*)ptr, desc, queue);
ptr = (void**)((char*)ptr + skip_size);
}
return p + 4;
int
mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
{
- return mono_threads_pthread_create (new_thread, attr, start_routine, arg);
+ return pthread_create (new_thread, attr, start_routine, arg);
}
int
LOCK_INIT (gc_mutex);
pagesize = mono_pagesize ();
- gc_debug_file = stdout;
+ gc_debug_file = stderr;
cb.thread_register = sgen_thread_register;
cb.thread_unregister = sgen_thread_unregister;
cb.thread_attach = sgen_thread_attach;
- cb.mono_method_is_critical = is_critical_method;
+ cb.mono_method_is_critical = (gpointer)is_critical_method;
+ cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
mono_threads_init (&cb, sizeof (SgenThreadInfo));
gc_debug_file = stderr;
g_free (rf);
}
+ } else if (!strcmp (opt, "print-allowance")) {
+ debug_print_allowance = TRUE;
} else if (!strcmp (opt, "collect-before-allocs")) {
collect_before_allocs = 1;
} else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
fprintf (stderr, " disable-major\n");
fprintf (stderr, " xdomain-checks\n");
fprintf (stderr, " clear-at-gc\n");
+ fprintf (stderr, " print-allowance\n");
exit (1);
}
}
mono_mb_emit_ldloc (mb, new_next_var);
mono_mb_emit_byte (mb, CEE_STIND_I);
+ /*The tlab store must be visible before the the vtable store. This could be replaced with a DDS but doing it with IL would be tricky. */
+ mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
+ mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
+
/* *p = vtable; */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_byte (mb, CEE_STIND_I);
}
+ /*
+ We must make sure both vtable and max_length are globaly visible before returning to managed land.
+ */
+ mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
+ mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
+
/* return p */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_byte (mb, CEE_RET);
return ATYPE_NUM;
}
+static void
+emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
+{
+ memset (nursery_check_return_labels, 0, sizeof (int) * 3);
+#ifdef SGEN_ALIGN_NURSERY
+ // if (ptr_in_nursery (ptr)) return;
+ /*
+ * Masking out the bits might be faster, but we would have to use 64 bit
+ * immediates, which might be slower.
+ */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_byte (mb, CEE_SHR_UN);
+ mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
+ nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
+
+ // if (!ptr_in_nursery (*ptr)) return;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_byte (mb, CEE_SHR_UN);
+ mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
+ nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
+#else
+ int label_continue1, label_continue2;
+ int dereferenced_var;
+
+ // if (ptr < (nursery_start)) goto continue;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_start);
+ label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
+
+ // if (ptr >= nursery_end)) goto continue;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_end);
+ label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
+
+ // Otherwise return
+ nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
+
+ // continue:
+ mono_mb_patch_branch (mb, label_continue_1);
+ mono_mb_patch_branch (mb, label_continue_2);
+
+ // Dereference and store in local var
+ dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_stloc (mb, dereferenced_var);
+
+ // if (*ptr < nursery_start) return;
+ mono_mb_emit_ldloc (mb, dereferenced_var);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_start);
+ nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
+
+ // if (*ptr >= nursery_end) return;
+ mono_mb_emit_ldloc (mb, dereferenced_var);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_end);
+ nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
+#endif
+}
MonoMethod*
mono_gc_get_write_barrier (void)
MonoMethodBuilder *mb;
MonoMethodSignature *sig;
#ifdef MANAGED_WBARRIER
- int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
-#ifndef SGEN_ALIGN_NURSERY
- int label_continue_1, label_continue_2, label_no_wb_5;
- int dereferenced_var;
-#endif
+ int i, nursery_check_labels [3];
+ int label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
int buffer_var, buffer_index_var, dummy_var;
+ gboolean use_managed_barrier;
#ifdef HAVE_KW_THREAD
int stack_end_offset = -1, store_remset_buffer_offset = -1;
#endif
#endif
- g_assert (!use_cardtable);
-
// FIXME: Maybe create a separate version for ctors (the branch would be
// correctly predicted more times)
if (write_barrier_method)
mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
#ifdef MANAGED_WBARRIER
- if (mono_runtime_has_tls_get ()) {
-#ifdef SGEN_ALIGN_NURSERY
- // if (ptr_in_nursery (ptr)) return;
+ use_managed_barrier = TRUE;
+#endif
+
+ if (use_managed_barrier && use_cardtable) {
+ emit_nursery_check (mb, nursery_check_labels);
/*
- * Masking out the bits might be faster, but we would have to use 64 bit
- * immediates, which might be slower.
- */
+ addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
+ *addr = 1;
+
+ sgen_cardtable:
+ LDC_PTR sgen_cardtable
+
+ address >> CARD_BITS
+ LDARG_0
+ LDC_I4 CARD_BITS
+ SHR_UN
+ if (SGEN_HAVE_OVERLAPPING_CARDS) {
+ LDC_PTR card_table_mask
+ AND
+ }
+ AND
+ ldc_i4_1
+ stind_i1
+ */
+ mono_mb_emit_ptr (mb, sgen_cardtable);
mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_icon (mb, CARD_BITS);
mono_mb_emit_byte (mb, CEE_SHR_UN);
- mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
- label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
-
- // if (!ptr_in_nursery (*ptr)) return;
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_byte (mb, CEE_LDIND_I);
- mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
- mono_mb_emit_byte (mb, CEE_SHR_UN);
- mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
- label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
-#else
-
- // if (ptr < (nursery_start)) goto continue;
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_ptr (mb, (gpointer) nursery_start);
- label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
-
- // if (ptr >= nursery_end)) goto continue;
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_ptr (mb, (gpointer) nursery_end);
- label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
-
- // Otherwise return
- label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
-
- // continue:
- mono_mb_patch_branch (mb, label_continue_1);
- mono_mb_patch_branch (mb, label_continue_2);
-
- // Dereference and store in local var
- dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_byte (mb, CEE_LDIND_I);
- mono_mb_emit_stloc (mb, dereferenced_var);
-
- // if (*ptr < nursery_start) return;
- mono_mb_emit_ldloc (mb, dereferenced_var);
- mono_mb_emit_ptr (mb, (gpointer) nursery_start);
- label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
+#ifdef SGEN_HAVE_OVERLAPPING_CARDS
+ mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
+ mono_mb_emit_byte (mb, CEE_AND);
+#endif
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_icon (mb, 1);
+ mono_mb_emit_byte (mb, CEE_STIND_I1);
- // if (*ptr >= nursery_end) return;
- mono_mb_emit_ldloc (mb, dereferenced_var);
- mono_mb_emit_ptr (mb, (gpointer) nursery_end);
- label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
+ // return;
+ for (i = 0; i < 3; ++i) {
+ if (nursery_check_labels [i])
+ mono_mb_patch_branch (mb, nursery_check_labels [i]);
+ }
+ mono_mb_emit_byte (mb, CEE_RET);
+ } else if (use_managed_barrier && mono_runtime_has_tls_get ()) {
+ emit_nursery_check (mb, nursery_check_labels);
-#endif
// if (ptr >= stack_end) goto need_wb;
mono_mb_emit_ldarg (mb, 0);
EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
mono_mb_emit_byte (mb, CEE_STIND_I);
// return;
- mono_mb_patch_branch (mb, label_no_wb_1);
- mono_mb_patch_branch (mb, label_no_wb_2);
+ for (i = 0; i < 3; ++i) {
+ if (nursery_check_labels [i])
+ mono_mb_patch_branch (mb, nursery_check_labels [i]);
+ }
mono_mb_patch_branch (mb, label_no_wb_3);
mono_mb_patch_branch (mb, label_no_wb_4);
-#ifndef SGEN_ALIGN_NURSERY
- mono_mb_patch_branch (mb, label_no_wb_5);
-#endif
mono_mb_emit_byte (mb, CEE_RET);
// slow path
mono_mb_patch_branch (mb, label_slow_path);
+
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
+ mono_mb_emit_byte (mb, CEE_RET);
+ } else {
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
+ mono_mb_emit_byte (mb, CEE_RET);
}
-#endif
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
- mono_mb_emit_byte (mb, CEE_RET);
res = mono_mb_create_method (mb, sig, 16);
mono_mb_free (mb);
projects / mono.git / blobdiff