* Author:
* Paolo Molaro (lupus@ximian.com)
*
- * Copyright (C) 2005-2006 Novell, Inc
+ * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
*
* Thread start/stop adapted from Boehm's GC:
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
}
*/
-#define MAX_DEBUG_LEVEL 9
+#define MAX_DEBUG_LEVEL 8
#define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
#define TV_DECLARE(name) struct timeval name
static NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
+/*
+ * If this is set, the nursery is aligned to an address aligned to its size, ie.
+ * a 1MB nursery will be aligned to an address divisible by 1MB. This allows us to
+ * speed up ptr_in_nursery () checks which are very frequent. This requires the
+ * nursery size to be a compile time constant.
+ */
+#define ALIGN_NURSERY 1
+
/*
* The young generation is divided into fragments. This is because
* we can hand one fragments to a thread for lock-less fast alloc and
((mword*)(obj))[0] &= ~PINNED_BIT; \
} while (0)
-#define ptr_in_nursery(ptr) ((char*)ptr >= nursery_start && (char*)ptr < nursery_real_end)
+#ifdef ALIGN_NURSERY
+#define ptr_in_nursery(ptr) (((mword)(ptr) & ~((1 << DEFAULT_NURSERY_BITS) - 1)) == (mword)nursery_start)
+#else
+#define ptr_in_nursery(ptr) ((char*)(ptr) >= nursery_start && (char*)(ptr) < nursery_real_end)
+#endif
/*
* Since we set bits in the vtable, use the macro to load it from the pointer to
/* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
//#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
#define DEFAULT_NURSERY_SIZE (1024*512*2)
+/* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
+#define DEFAULT_NURSERY_BITS 20
#define DEFAULT_MAX_SECTION (DEFAULT_NURSERY_SIZE * 16)
#define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
/* to quickly find the head of an object pinned by a conservative address
/*
* used when moving the objects
* When the nursery is collected, objects are copied to to_space.
- * The area between to_space and gray_objects is used as a stack
+ * The area between gray_first and gray_objects is used as a stack
* of objects that need their fields checked for more references
* to be copied.
* We should optimize somehow this mechanism to avoid rescanning
* test cache misses and other graph traversal orders.
*/
static char *to_space = NULL;
+static char *gray_first = NULL;
static char *gray_objects = NULL;
static char *to_space_end = NULL;
static GCMemSection *to_space_section = NULL;
return NULL;
}
+/*
+ * drain_gray_stack:
+ *
+ * Scan objects in the gray stack until the stack is empty. This should be called
+ * frequently after each object is copied, to achieve better locality and cache
+ * usage.
+ */
+static void inline
+drain_gray_stack (char *start_addr, char *end_addr)
+{
+ char *gray_start = gray_first;
+
+ while (gray_start < gray_objects) {
+ DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
+ gray_start = scan_object (gray_start, start_addr, end_addr);
+ }
+
+ gray_first = gray_start;
+}
+
/*
* scan_vtype:
*
while (desc) {
if ((desc & 1) && *start_root) {
*start_root = copy_object (*start_root, n_start, n_end);
- DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root)); \
+ DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
+ drain_gray_stack (n_start, n_end);
}
desc >>= 1;
start_root++;
if ((bmap & 1) && *objptr) {
*objptr = copy_object (*objptr, n_start, n_end);
DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
+ drain_gray_stack (n_start, n_end);
}
bmap >>= 1;
++objptr;
char *data;
int scan_starts;
Fragment *frag;
+ int alloc_size;
if (nursery_section)
return;
*/
/* FIXME: handle OOM */
section = get_internal_mem (sizeof (GCMemSection));
- data = get_os_memory (nursery_size, TRUE);
- nursery_start = nursery_next = data;
- nursery_real_end = data + nursery_size;
+
+#ifdef ALIGN_NURSERY
+ /* Allocate twice the memory to be able to put the nursery at an aligned address */
+ g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
+
+ alloc_size = nursery_size * 2;
+ data = get_os_memory (alloc_size, TRUE);
+ nursery_start = (void*)(((mword)data + (1 << DEFAULT_NURSERY_BITS) - 1) & ~((1 << DEFAULT_NURSERY_BITS) - 1));
+ g_assert ((char*)nursery_start + nursery_size <= ((char*)data + alloc_size));
+ /* FIXME: Use the remaining size for something else, if it is big enough */
+#else
+ alloc_size = nursery_size;
+ data = get_os_memory (alloc_size, TRUE);
+ nursery_start = data;
+#endif
+ nursery_real_end = nursery_start + nursery_size;
UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end);
- total_alloc += nursery_size;
+ nursery_next = nursery_start;
+ total_alloc += alloc_size;
DEBUG (4, fprintf (gc_debug_file, "Expanding heap size: %zd, total: %zd\n", nursery_size, total_alloc));
section->data = section->next_data = data;
- section->size = nursery_size;
+ section->size = alloc_size;
section->end_data = nursery_real_end;
- scan_starts = nursery_size / SCAN_START_SIZE;
+ scan_starts = alloc_size / SCAN_START_SIZE;
section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts);
section->num_scan_start = scan_starts;
section->role = MEMORY_ROLE_GEN0;
}
static void
-drain_gray_stack (char *start_addr, char *end_addr)
+finish_gray_stack (char *start_addr, char *end_addr)
{
TV_DECLARE (atv);
TV_DECLARE (btv);
* We need to walk the LO list as well in search of marked big objects
* (use a flag since this is needed only on major collections). We need to loop
* here as well, so keep a counter of marked LO (increasing it in copy_object).
+ * To achieve better cache locality and cache usage, we drain the gray stack
+ * frequently, after each object is copied, and just finish the work here.
*/
- TV_GETTIME (btv);
- gray_start = to_space;
- DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area: %p-%p, size: %d\n", gray_start, gray_objects, (int)(gray_objects - gray_start)));
+ gray_start = gray_first;
while (gray_start < gray_objects) {
DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start)));
gray_start = scan_object (gray_start, start_addr, end_addr);
}
TV_GETTIME (atv);
- DEBUG (2, fprintf (gc_debug_file, "Gray stack scan: %d usecs\n", TV_ELAPSED (btv, atv)));
//scan_old_generation (start_addr, end_addr);
DEBUG (2, fprintf (gc_debug_file, "Old generation done\n"));
/* walk the finalization queue and move also the objects that need to be
* We reset to_space if we allocated objects in degraded mode.
*/
if (to_space_section)
- to_space = gray_objects = to_space_section->next_data;
+ to_space = gray_objects = gray_first = to_space_section->next_data;
if ((to_space_end - to_space) < max_garbage_amount) {
section = alloc_section (nursery_section->size * 4);
g_assert (nursery_section->size >= max_garbage_amount);
- to_space = gray_objects = section->next_data;
+ to_space = gray_objects = gray_first = section->next_data;
to_space_end = section->end_data;
to_space_section = section;
}
TV_GETTIME (btv);
DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
- drain_gray_stack (nursery_start, nursery_next);
+ finish_gray_stack (nursery_start, nursery_next);
/* walk the pin_queue, build up the fragment list of free memory, unmark
* pinned objects as we go, memzero() the empty fragments so they are ready for the
/* allocate the big to space */
DEBUG (4, fprintf (gc_debug_file, "Allocate tospace for size: %zd\n", copy_space_required));
section = alloc_section (copy_space_required);
- to_space = gray_objects = section->next_data;
+ to_space = gray_objects = gray_first = section->next_data;
to_space_end = section->end_data;
to_space_section = section;
/* we need to go over the big object list to see if any was marked and scan it
* And we need to make this in a loop, considering that objects referenced by finalizable
- * objects could reference big objects (this happens in drain_gray_stack ())
+ * objects could reference big objects (this happens in finish_gray_stack ())
*/
scan_needed_big_objects (heap_start, heap_end);
/* all the objects in the heap */
- drain_gray_stack (heap_start, heap_end);
+ finish_gray_stack (heap_start, heap_end);
/* sweep the big objects list */
prevbo = NULL;
mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
{
RememberedSet *rs;
- if ((char*)field_ptr >= nursery_start && (char*)field_ptr < nursery_real_end) {
+ if (ptr_in_nursery (field_ptr)) {
*(void**)field_ptr = value;
return;
}
mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
{
RememberedSet *rs = remembered_set;
- if ((char*)slot_ptr >= nursery_start && (char*)slot_ptr < nursery_real_end) {
+ if (ptr_in_nursery (slot_ptr)) {
*(void**)slot_ptr = value;
return;
}
mono_gc_wbarrier_arrayref_copy (MonoArray *arr, gpointer slot_ptr, int count)
{
RememberedSet *rs = remembered_set;
- if ((char*)slot_ptr >= nursery_start && (char*)slot_ptr < nursery_real_end)
+ if (ptr_in_nursery (slot_ptr))
return;
DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", slot_ptr, count));
if (rs->store_next + 1 < rs->end_set) {
mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
{
RememberedSet *rs;
- if ((char*)ptr >= nursery_start && (char*)ptr < nursery_real_end) {
+ if (ptr_in_nursery (ptr)) {
DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
*(void**)ptr = value;
return;
mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
{
RememberedSet *rs = remembered_set;
- if ((char*)ptr >= nursery_start && (char*)ptr < nursery_real_end) {
+ if (ptr_in_nursery (ptr))
return;
- }
DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
if (rs->store_next + 2 < rs->end_set) {
mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
{
RememberedSet *rs = remembered_set;
- if ((char*)dest >= nursery_start && (char*)dest < nursery_real_end) {
+ if (ptr_in_nursery (dest))
return;
- }
DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d for class %s\n", dest, count, klass->name));
if (rs->store_next + 2 < rs->end_set) {
mword desc;
int type;
- if ((ptr >= nursery_start) && (ptr < nursery_real_end)) {
+ if (ptr_in_nursery (ptr)) {
printf ("Pointer inside nursery.\n");
} else {
for (section = section_list; section;) {
printf ("VTable is invalid (empty).\n");
return;
}
- if (((char*)vtable >= nursery_start) && ((char*)vtable < nursery_real_end)) {
+ if (ptr_in_nursery (vtable)) {
printf ("VTable is invalid (points inside nursery).\n");
return;
}
}
return p + 3;
+ case REMSET_ROOT_LOCATION:
+ return p + 2;
+ default:
+ g_assert_not_reached ();
}
break;
}
int
mono_gc_get_generation (MonoObject *obj)
{
- if ((char*)obj >= nursery_start && (char*)obj < nursery_real_end)
+ if (ptr_in_nursery (obj))
return 0;
return 1;
}
return ATYPE_NUM;
}
+static MonoMethod *write_barrier_method;
+
+MonoMethod*
+mono_gc_get_write_barrier (void)
+{
+ MonoMethod *res;
+ int remset_offset = -1;
+ int remset_var, next_var;
+ MonoMethodBuilder *mb;
+ MonoMethodSignature *sig;
+ int label1, label2;
+
+ MONO_THREAD_VAR_OFFSET (remembered_set, remset_offset);
+
+ // FIXME: Maybe create a separate version for ctors (the branch would be
+ // correctly predicted more times)
+ if (write_barrier_method)
+ return write_barrier_method;
+
+ /* Create the IL version of mono_gc_barrier_generic_store () */
+ sig = mono_metadata_signature_alloc (mono_defaults.corlib, 2);
+ sig->ret = &mono_defaults.void_class->byval_arg;
+ sig->params [0] = &mono_defaults.int_class->byval_arg;
+ sig->params [1] = &mono_defaults.object_class->byval_arg;
+
+ mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
+
+ /* ptr_in_nursery () check */
+#ifdef ALIGN_NURSERY
+ /*
+ * Masking out the bits might be faster, but we would have to use 64 bit
+ * immediates, which might be slower.
+ */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_byte (mb, CEE_SHR_UN);
+ mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
+ label1 = mono_mb_emit_branch (mb, CEE_BNE_UN);
+#else
+ // FIXME:
+ g_assert_not_reached ();
+#endif
+
+ /* Don't need write barrier case */
+ /* do the assignment */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldarg (mb, 1);
+ /* Don't use STIND_REF, as it would cause infinite recursion */
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ /* Need write barrier case */
+ mono_mb_patch_branch (mb, label1);
+
+ if (remset_offset == -1)
+ // FIXME:
+ g_assert_not_reached ();
+
+ // remset_var = remembered_set;
+ remset_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+ mono_mb_emit_byte (mb, CEE_MONO_TLS);
+ mono_mb_emit_i4 (mb, remset_offset);
+ mono_mb_emit_stloc (mb, remset_var);
+
+ // next_var = rs->store_next
+ next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldloc (mb, remset_var);
+ mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, store_next));
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_stloc (mb, next_var);
+
+ // if (rs->store_next < rs->end_set) {
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_ldloc (mb, remset_var);
+ mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, end_set));
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ label2 = mono_mb_emit_branch (mb, CEE_BGE);
+
+ /* write barrier fast path */
+ // *(rs->store_next++) = (mword)ptr;
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_icon (mb, sizeof (gpointer));
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_stloc (mb, next_var);
+
+ mono_mb_emit_ldloc (mb, remset_var);
+ mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, store_next));
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ // *(void**)ptr = value;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldarg (mb, 1);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ /* write barrier slow path */
+ mono_mb_patch_branch (mb, label2);
+
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldarg (mb, 1);
+ mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_store);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ res = mono_mb_create_method (mb, sig, 16);
+ mono_mb_free (mb);
+
+ mono_loader_lock ();
+ if (write_barrier_method) {
+ /* Already created */
+ mono_free_method (res);
+ } else {
+ /* double-checked locking */
+ mono_memory_barrier ();
+ write_barrier_method = res;
+ }
+ mono_loader_unlock ();
+
+ return write_barrier_method;
+}
+
#endif /* HAVE_SGEN_GC */
projects / mono.git / blobdiff