X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Fmetadata%2Fsgen-gc.c;h=3bd39a4ea8bd0e039738dee02dba4f33d8f0128b;hb=59ce8a23634c6ea6027696a91804d57df0d8a900;hp=af2b34d35af3f556c323fa2fcae6317d99b3eb18;hpb=40967aa9d98e641cc93041c4fc03271c071b7be3;p=mono.git diff --git a/mono/metadata/sgen-gc.c b/mono/metadata/sgen-gc.c index af2b34d35af..3bd39a4ea8b 100644 --- a/mono/metadata/sgen-gc.c +++ b/mono/metadata/sgen-gc.c @@ -127,7 +127,6 @@ #include #include #include -#include #include #include #include @@ -145,7 +144,11 @@ #include "metadata/threads.h" #include "metadata/sgen-gc.h" #include "metadata/mono-gc.h" +#include "utils/mono-mmap.h" +#ifdef HAVE_VALGRIND_MEMCHECK_H +#include +#endif /* * ###################################################################### @@ -224,7 +227,8 @@ struct _LOSObject { LOSObject *next; mword size; /* this is the object size */ int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */ - int role; + guint16 role; + guint16 scanned; char data [MONO_ZERO_LEN_ARRAY]; }; @@ -325,6 +329,7 @@ enum { }; static __thread RememberedSet *remembered_set MONO_TLS_FAST; +static pthread_key_t remembered_set_key; static RememberedSet *global_remset; static int store_to_global_remset = 0; @@ -345,7 +350,7 @@ typedef struct { /* these bits are set in the object vtable: we could merge them since an object can be * either pinned or forwarded but not both. * We store them in the vtable slot because the bits are used in the sync block for - * other purpouses: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change + * other purposes: if we merge them and alloc the sync blocks aligned to 8 bytes, we can change * this and use bit 3 in the syncblock (with the lower two bits both set for forwarded, that * would be an invalid combination for the monitor and hash code). * The values are already shifted. @@ -406,12 +411,25 @@ safe_object_get_size (MonoObject* o) } } +static inline gboolean +is_half_constructed (MonoObject *o) +{ + MonoClass *klass; + + klass = ((MonoVTable*)LOAD_VTABLE (o))->klass; + if ((klass == mono_defaults.string_class && mono_string_length ((MonoString*)o) == 0) || + (klass->rank && mono_array_length ((MonoArray*)o) == 0)) + return TRUE; + else + return FALSE; +} + /* * ###################################################################### * ######## Global data. * ###################################################################### */ -static pthread_mutex_t gc_mutex = PTHREAD_MUTEX_INITIALIZER; +static LOCK_DECLARE (gc_mutex); static int gc_disabled = 0; static int num_minor_gcs = 0; static int num_major_gcs = 0; @@ -421,7 +439,7 @@ static int num_major_gcs = 0; #define DEFAULT_NURSERY_SIZE (1024*512*2) #define DEFAULT_MAX_SECTION (DEFAULT_NURSERY_SIZE * 16) #define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2) -/* to quickly find the heard of an object pinned by a conservative address +/* to quickly find the head of an object pinned by a conservative address * we keep track of the objects allocated for each SCAN_START_SIZE memory * chunk in the nursery or other memory sections. Larger values have less * memory overhead and bigger runtime cost. 4-8 KB are reasonable values. @@ -432,7 +450,7 @@ static int num_major_gcs = 0; /* This is a fixed value used for pinned chunks, not the system pagesize */ #define FREELIST_PAGESIZE 4096 -static mword pagesize = 4096; /* FIXME */ +static mword pagesize = 4096; static mword nursery_size = DEFAULT_NURSERY_SIZE; static mword next_section_size = DEFAULT_NURSERY_SIZE * 4; static mword max_section_size = DEFAULT_MAX_SECTION; @@ -475,9 +493,9 @@ static FinalizeEntry **disappearing_link_hash = NULL; static mword disappearing_link_hash_size = 0; static mword finalizable_hash_size = 0; -static mword num_registered_finalizers = 0; -static mword num_ready_finalizers = 0; -static mword num_disappearing_links = 0; +static int num_registered_finalizers = 0; +static int num_ready_finalizers = 0; +static int num_disappearing_links = 0; static int no_finalize = 0; /* keep each size a multiple of ALLOC_ALIGN */ @@ -510,7 +528,7 @@ obj_is_from_pinned_alloc (char *p) static RootRecord **roots_hash = NULL; static int roots_hash_size = 0; static mword roots_size = 0; /* amount of memory in the root set */ -static mword num_roots_entries = 0; +static int num_roots_entries = 0; /* * The current allocation cursors @@ -566,12 +584,6 @@ static GCMemSection *to_space_section = NULL; * ###################################################################### */ -/* - * Recursion is not allowed for the thread lock. - */ -#define LOCK_GC pthread_mutex_lock (&gc_mutex) -#define UNLOCK_GC pthread_mutex_unlock (&gc_mutex) - #define UPDATE_HEAP_BOUNDARIES(low,high) do { \ if ((mword)(low) < lowest_heap_address) \ lowest_heap_address = (mword)(low); \ @@ -602,8 +614,8 @@ static void scan_from_remsets (void *start_nursery, void *end_nursery); static void find_pinning_ref_from_thread (char *obj, size_t size); static void update_current_thread_stack (void *start); static GCMemSection* alloc_section (size_t size); -static void finalize_in_range (void **start, void **end); -static void null_link_in_range (void **start, void **end); +static void finalize_in_range (char *start, char *end); +static void null_link_in_range (char *start, char *end); static gboolean search_fragment_for_size (size_t size); static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end); static void clear_remsets (void); @@ -611,6 +623,8 @@ static void sweep_pinned_objects (void); static void free_large_object (LOSObject *obj); static void free_mem_section (GCMemSection *section); +void check_consistency (void); + /* * ###################################################################### * ######## GC descriptors @@ -737,7 +751,7 @@ alloc_complex_descriptor (gsize *bitmap, int numbits) * Descriptor builders. */ void* -mono_gc_make_descr_for_string (void) +mono_gc_make_descr_for_string (gsize *bitmap, int numbits) { return (void*) DESC_TYPE_STRING; } @@ -765,24 +779,24 @@ mono_gc_make_descr_for_object (gsize *bitmap, int numbits, size_t obj_size) */ if (first_set < 0) { desc = DESC_TYPE_RUN_LENGTH | stored_size; - DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %d\n", (void*)desc, stored_size)); + DEBUG (6, fprintf (gc_debug_file, "Ptrfree descriptor %p, size: %zd\n", (void*)desc, stored_size)); return (void*) desc; } else if (first_set < 256 && num_set < 256 && (first_set + num_set == last_set + 1)) { desc = DESC_TYPE_RUN_LENGTH | stored_size | (first_set << 16) | (num_set << 24); - DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %d, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set)); + DEBUG (6, fprintf (gc_debug_file, "Runlen descriptor %p, size: %zd, first set: %d, num set: %d\n", (void*)desc, stored_size, first_set, num_set)); return (void*) desc; } /* we know the 2-word header is ptr-free */ if (last_set < SMALL_BITMAP_SIZE + OBJECT_HEADER_WORDS) { desc = DESC_TYPE_SMALL_BITMAP | stored_size | ((*bitmap >> OBJECT_HEADER_WORDS) << SMALL_BITMAP_SHIFT); - DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %d, last set: %d\n", (void*)desc, stored_size, last_set)); + DEBUG (6, fprintf (gc_debug_file, "Smallbitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set)); return (void*) desc; } } /* we know the 2-word header is ptr-free */ if (last_set < LARGE_BITMAP_SIZE + OBJECT_HEADER_WORDS) { desc = DESC_TYPE_LARGE_BITMAP | ((*bitmap >> OBJECT_HEADER_WORDS) << LOW_TYPE_BITS); - DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %d, last set: %d\n", (void*)desc, stored_size, last_set)); + DEBUG (6, fprintf (gc_debug_file, "Largebitmap descriptor %p, size: %zd, last set: %d\n", (void*)desc, stored_size, last_set)); return (void*) desc; } /* it's a complex object ... */ @@ -811,7 +825,7 @@ mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_ } /* Note: we also handle structs with just ref fields */ if (num_set * sizeof (gpointer) == elem_size) { - return (void*)(desc | VECTOR_SUBTYPE_REFS | ((-1LL) << 16)); + return (void*)(desc | VECTOR_SUBTYPE_REFS | ((gssize)(-1) << 16)); } /* FIXME: try run-len first */ /* Note: we can't skip the object header here, because it's not present */ @@ -898,7 +912,7 @@ mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_ bitmap_data++; \ if (0) { \ MonoObject *myobj = (MonoObject*)obj; \ - g_print ("found %d at %p (0x%x): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \ + g_print ("found %d at %p (0x%zx): %s.%s\n", bwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \ } \ while (bwords-- > 0) { \ gsize _bmap = *bitmap_data++; \ @@ -925,7 +939,7 @@ mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, int numbits, size_ char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \ if (0) { \ MonoObject *myobj = (MonoObject*)start; \ - g_print ("found %d at %p (0x%x): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \ + g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \ } \ while (e_start < e_end) { \ void **_objptr = (void**)e_start; \ @@ -1034,7 +1048,7 @@ scan_area (char *start, char *end) DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name)); if (0) { MonoObject *obj = (MonoObject*)start; - g_print ("found at %p (0x%x): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name); + g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name); } type = vt->desc & 0x7; if (type == DESC_TYPE_STRING) { @@ -1251,7 +1265,7 @@ add_to_global_remset (gpointer ptr) static char* __attribute__((noinline)) copy_object (char *obj, char *from_space_start, char *from_space_end) { - if (obj >= from_space_start && obj < from_space_end) { + if (obj >= from_space_start && obj < from_space_end && (obj < to_space || obj >= to_space_end)) { MonoVTable *vt; char *forwarded; mword objsize; @@ -1269,7 +1283,7 @@ copy_object (char *obj, char *from_space_start, char *from_space_end) objsize = safe_object_get_size ((MonoObject*)obj); objsize += ALLOC_ALIGN - 1; objsize &= ~(ALLOC_ALIGN - 1); - DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %d)\n", gray_objects, ((MonoObject*)obj)->vtable->klass->name, objsize)); + DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", gray_objects, ((MonoObject*)obj)->vtable->klass->name, objsize)); /* FIXME: handle pinned allocs: * Large objects are simple, at least until we always follow the rule: * if objsize >= MAX_SMALL_OBJ_SIZE, pin the object and return it. @@ -1277,7 +1291,7 @@ copy_object (char *obj, char *from_space_start, char *from_space_end) * the object is pinned, it is marked, otherwise it can be freed. */ if (objsize >= MAX_SMALL_OBJ_SIZE || (obj >= min_pinned_chunk_addr && obj < max_pinned_chunk_addr && obj_is_from_pinned_alloc (obj))) { - DEBUG (9, fprintf (gc_debug_file, "Marked LOS/Pinned %p (%s), size: %d\n", obj, safe_name (obj), objsize)); + DEBUG (9, fprintf (gc_debug_file, "Marked LOS/Pinned %p (%s), size: %zd\n", obj, safe_name (obj), objsize)); pin_object (obj); return obj; } @@ -1304,7 +1318,7 @@ copy_object (char *obj, char *from_space_start, char *from_space_end) if (vt->rank && ((MonoArray*)obj)->bounds) { MonoArray *array = (MonoArray*)gray_objects; array->bounds = (MonoArrayBounds*)((char*)gray_objects + ((char*)((MonoArray*)obj)->bounds - (char*)obj)); - DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %d, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array))); + DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array))); } /* set the forwarding pointer */ forward_object (obj, gray_objects); @@ -1462,7 +1476,7 @@ pin_objects_from_addresses (GCMemSection *section, void **start, void **end, voi last_obj_size = safe_object_get_size ((MonoObject*)search_start); last_obj_size += ALLOC_ALIGN - 1; last_obj_size &= ~(ALLOC_ALIGN - 1); - DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %d\n", last_obj, safe_name (last_obj), last_obj_size)); + DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size)); if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) { DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count)); pin_object (search_start); @@ -1545,11 +1559,11 @@ print_nursery_gaps (void* start_nursery, void *end_nursery) gpointer next; for (i = 0; i < next_pin_slot; ++i) { next = pin_queue [i]; - fprintf (gc_debug_file, "Nursery range: %p-%p, size: %d\n", first, next, (char*)next-(char*)first); + fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first); first = next; } next = end_nursery; - fprintf (gc_debug_file, "Nursery range: %p-%p, size: %d\n", first, next, (char*)next-(char*)first); + fprintf (gc_debug_file, "Nursery range: %p-%p, size: %zd\n", first, next, (char*)next-(char*)first); } /* reduce the info in the pin queue, removing duplicate pointers and sorting them */ @@ -1627,6 +1641,15 @@ conservatively_pin_objects_from (void **start, void **end, void *start_nursery, start++; } DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count)); + +#ifdef HAVE_VALGRIND_MEMCHECK_H + /* + * The pinning addresses might come from undefined memory, this is normal. Since they + * are used in lots of functions, we make the memory defined here instead of having + * to add a supression for those functions. + */ + VALGRIND_MAKE_MEM_DEFINED (pin_queue, next_pin_slot * sizeof (pin_queue [0])); +#endif } /* @@ -1760,7 +1783,7 @@ alloc_nursery (void) if (nursery_section) return; - DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %d\n", nursery_size)); + DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %zd\n", nursery_size)); /* later we will alloc a larger area for the nursery but only activate * what we need. The rest will be used as expansion if we have too many pinned * objects in the existing nursery. @@ -1773,7 +1796,7 @@ alloc_nursery (void) nursery_temp_end = data + SCAN_START_SIZE; UPDATE_HEAP_BOUNDARIES (nursery_start, nursery_real_end); total_alloc += nursery_size; - DEBUG (4, fprintf (gc_debug_file, "Expanding heap size: %d, total: %d\n", nursery_size, total_alloc)); + DEBUG (4, fprintf (gc_debug_file, "Expanding heap size: %zd, total: %zd\n", nursery_size, total_alloc)); section->data = section->next_data = data; section->size = nursery_size; section->end_data = nursery_real_end; @@ -1825,7 +1848,7 @@ scan_old_generation (char *start, char* end) } /* scan the old object space, too */ for (big_object = los_object_list; big_object; big_object = big_object->next) { - DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %d\n", big_object->data, safe_name (big_object->data), big_object->size)); + DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size)); scan_object (big_object->data, start, end); } /* scan the list of objects ready for finalization */ @@ -1845,7 +1868,7 @@ static void add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end) { Fragment *fragment; - DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %d\n", frag_start, frag_end, frag_size)); + DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size)); /* memsetting just the first chunk start is bound to provide better cache locality */ memset (frag_start, 0, frag_size); /* Not worth dealing with smaller fragments: need to tune */ @@ -1860,12 +1883,28 @@ add_nursery_frag (size_t frag_size, char* frag_start, char* frag_end) } } +static int +scan_needed_big_objects (char *start_addr, char *end_addr) +{ + LOSObject *big_object; + int count = 0; + for (big_object = los_object_list; big_object; big_object = big_object->next) { + if (!big_object->scanned && object_is_pinned (big_object->data)) { + DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size)); + scan_object (big_object->data, start_addr, end_addr); + big_object->scanned = TRUE; + count++; + } + } + return count; +} + static void drain_gray_stack (char *start_addr, char *end_addr) { TV_DECLARE (atv); TV_DECLARE (btv); - int fin_ready; + int fin_ready, bigo_scanned_num; char *gray_start; /* @@ -1899,7 +1938,8 @@ drain_gray_stack (char *start_addr, char *end_addr) */ do { fin_ready = num_ready_finalizers; - finalize_in_range ((void**)start_addr, (void**)end_addr); + finalize_in_range (start_addr, end_addr); + bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr); /* drain the new stack that might have been created */ DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin: %p-%p, size: %d\n", gray_start, gray_objects, (int)(gray_objects - gray_start))); @@ -1907,7 +1947,7 @@ drain_gray_stack (char *start_addr, char *end_addr) DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", gray_start, safe_name (gray_start))); gray_start = scan_object (gray_start, start_addr, end_addr); } - } while (fin_ready != num_ready_finalizers); + } while (fin_ready != num_ready_finalizers || bigo_scanned_num); DEBUG (2, fprintf (gc_debug_file, "Copied to old space: %d bytes\n", (int)(gray_objects - to_space))); to_space = gray_start; @@ -1921,7 +1961,7 @@ drain_gray_stack (char *start_addr, char *end_addr) * GC a finalized object my lose the monitor because it is cleared before the finalizer is * called. */ - null_link_in_range ((void**)start_addr, (void**)end_addr); + null_link_in_range (start_addr, end_addr); TV_GETTIME (btv); DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan: %d usecs\n", TV_ELAPSED (atv, btv))); } @@ -1929,7 +1969,7 @@ drain_gray_stack (char *start_addr, char *end_addr) static int last_num_pinned = 0; static void -build_nursery_fragments (int start_pin, int end_pin) +build_nursery_fragments (int start_pin, int end_pin, char *nursery_last_allocated) { char *frag_start, *frag_end; size_t frag_size; @@ -1954,6 +1994,17 @@ build_nursery_fragments (int start_pin, int end_pin) frag_size += ALLOC_ALIGN - 1; frag_size &= ~(ALLOC_ALIGN - 1); frag_start = (char*)pin_queue [i] + frag_size; + /* + * pin_queue [i] might point to a half-constructed string or vector whose + * length field is not set. In that case, frag_start points inside the + * (zero initialized) object. Find the end of the object by scanning forward. + * + */ + if (is_half_constructed (pin_queue [i])) { + /* Can't use nursery_next as the limit as it is modified in collect_nursery () */ + while ((frag_start < nursery_last_allocated) && *(mword*)frag_start == 0) + frag_start += sizeof (mword); + } } nursery_last_pinned_end = frag_start; frag_end = nursery_real_end; @@ -1988,7 +2039,11 @@ build_section_fragments (GCMemSection *section) frag_end = pin_queue [i]; /* remove the pin bit from pinned objects */ unpin_object (frag_end); - section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end; + if (frag_end >= section->data + section->size) { + frag_end = section->data + section->size; + } else { + section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end; + } frag_size = frag_end - frag_start; if (frag_size) memset (frag_start, 0, frag_size); @@ -2004,6 +2059,22 @@ build_section_fragments (GCMemSection *section) memset (frag_start, 0, frag_size); } +static void +scan_from_registered_roots (char *addr_start, char *addr_end) +{ + int i; + RootRecord *root; + for (i = 0; i < roots_hash_size; ++i) { + for (root = roots_hash [i]; root; root = root->next) { + /* if desc is non-null it has precise info */ + if (!root->root_desc) + continue; + DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc)); + precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc); + } + } +} + /* * Collect objects in the nursery. */ @@ -2013,11 +2084,14 @@ collect_nursery (size_t requested_size) GCMemSection *section; size_t max_garbage_amount; int i; - RootRecord *root; + char *nursery_last_allocated; + TV_DECLARE (all_atv); + TV_DECLARE (all_btv); TV_DECLARE (atv); TV_DECLARE (btv); degraded_mode = 0; + nursery_last_allocated = nursery_next; nursery_next = MAX (nursery_next, nursery_last_pinned_end); /* FIXME: optimize later to use the higher address where an object can be present */ nursery_next = MAX (nursery_next, nursery_real_end); @@ -2042,7 +2116,9 @@ collect_nursery (size_t requested_size) nursery_section->next_data = nursery_next; num_minor_gcs++; + mono_stats.minor_gc_count ++; /* world must be stopped already */ + TV_GETTIME (all_atv); TV_GETTIME (atv); /* pin from pinned handles */ pin_from_roots (nursery_start, nursery_next); @@ -2070,15 +2146,7 @@ collect_nursery (size_t requested_size) scan_object (pin_queue [i], nursery_start, nursery_next); } /* registered roots, this includes static fields */ - for (i = 0; i < roots_hash_size; ++i) { - for (root = roots_hash [i]; root; root = root->next) { - /* if desc is non-null it has precise info */ - if (!root->root_desc) - continue; - DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc)); - precisely_scan_objects_from ((void**)root->start_root, root->end_root, nursery_start, nursery_next, root->root_desc); - } - } + scan_from_registered_roots (nursery_start, nursery_next); TV_GETTIME (btv); DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv))); @@ -2088,9 +2156,12 @@ collect_nursery (size_t requested_size) * pinned objects as we go, memzero() the empty fragments so they are ready for the * next allocations. */ - build_nursery_fragments (0, next_pin_slot); + build_nursery_fragments (0, next_pin_slot, nursery_last_allocated); TV_GETTIME (atv); - DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %d bytes available\n", TV_ELAPSED (btv, atv), fragment_total)); + DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total)); + + TV_GETTIME (all_btv); + mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv); /* prepare the pin queue for the next collection */ last_num_pinned = next_pin_slot; @@ -2107,10 +2178,11 @@ major_collection (void) GCMemSection *section, *prev_section; LOSObject *bigobj, *prevbo; int i; - RootRecord *root; PinnedChunk *chunk; FinalizeEntry *fin; int count; + TV_DECLARE (all_atv); + TV_DECLARE (all_btv); TV_DECLARE (atv); TV_DECLARE (btv); /* FIXME: only use these values for the precise scan @@ -2123,6 +2195,7 @@ major_collection (void) degraded_mode = 0; DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs)); num_major_gcs++; + mono_stats.major_gc_count ++; /* * FIXME: implement Mark/Compact * Until that is done, we can just apply mostly the same alg as for the nursery: @@ -2133,6 +2206,7 @@ major_collection (void) collect_nursery (0); return; } + TV_GETTIME (all_atv); /* FIXME: make sure the nursery next_data ptr is updated */ nursery_section->next_data = nursery_real_end; /* we should also coalesce scanning from sections close to each other @@ -2144,7 +2218,7 @@ major_collection (void) TV_GETTIME (atv); DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n")); for (section = section_list; section; section = section->next) { - section->pin_queue_start = count = next_pin_slot; + section->pin_queue_start = count = section->pin_queue_end = next_pin_slot; pin_from_roots (section->data, section->next_data); if (count != next_pin_slot) { int reduced_to; @@ -2164,7 +2238,7 @@ major_collection (void) if (next_pin_slot != count) { next_pin_slot = count; pin_object (bigobj->data); - DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %d from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size)); + DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size)); } } /* look for pinned addresses for pinned-alloc objects */ @@ -2184,7 +2258,7 @@ major_collection (void) DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot)); /* allocate the big to space */ - DEBUG (4, fprintf (gc_debug_file, "Allocate tospace for size: %d\n", copy_space_required)); + DEBUG (4, fprintf (gc_debug_file, "Allocate tospace for size: %zd\n", copy_space_required)); section = alloc_section (copy_space_required); to_space = gray_objects = section->next_data; to_space_end = section->end_data; @@ -2196,21 +2270,14 @@ major_collection (void) * mark any section without pinned objects, so we can free it since we will be able to * move all the objects. */ - /* the pinned objects are roots */ + /* the pinned objects are roots (big objects are included in this list, too) */ for (i = 0; i < next_pin_slot; ++i) { DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i]))); scan_object (pin_queue [i], heap_start, heap_end); } /* registered roots, this includes static fields */ - for (i = 0; i < roots_hash_size; ++i) { - for (root = roots_hash [i]; root; root = root->next) { - /* if desc is non-null it has precise info */ - if (!root->root_desc) - continue; - DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc)); - precisely_scan_objects_from ((void**)root->start_root, root->end_root, heap_start, heap_end, root->root_desc); - } - } + scan_from_registered_roots (heap_start, heap_end); + /* scan the list of objects ready for finalization */ for (fin = fin_ready_list; fin; fin = fin->next) { DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object))); @@ -2219,6 +2286,11 @@ major_collection (void) TV_GETTIME (atv); DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv))); + /* we need to go over the big object list to see if any was marked and scan it + * And we need to make this in a loop, considering that objects referenced by finalizable + * objects could reference big objects (this happens in drain_gray_stack ()) + */ + scan_needed_big_objects (heap_start, heap_end); /* all the objects in the heap */ drain_gray_stack (heap_start, heap_end); @@ -2227,6 +2299,7 @@ major_collection (void) for (bigobj = los_object_list; bigobj;) { if (object_is_pinned (bigobj->data)) { unpin_object (bigobj->data); + bigobj->scanned = FALSE; } else { LOSObject *to_free; /* not referenced anywhere, so we can free it */ @@ -2277,8 +2350,10 @@ major_collection (void) * pinned objects as we go, memzero() the empty fragments so they are ready for the * next allocations. */ - build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end); + build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end, nursery_next); + TV_GETTIME (all_btv); + mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv); /* prepare the pin queue for the next collection */ next_pin_slot = 0; if (fin_ready_list) { @@ -2317,7 +2392,7 @@ alloc_section (size_t size) section->end_data = data + new_size; UPDATE_HEAP_BOUNDARIES (data, section->end_data); total_alloc += new_size; - DEBUG (2, fprintf (gc_debug_file, "Expanding heap size: %d, total: %d\n", new_size, total_alloc)); + DEBUG (2, fprintf (gc_debug_file, "Expanding heap size: %zd, total: %zd\n", new_size, total_alloc)); section->data = data; section->size = new_size; scan_starts = new_size / SCAN_START_SIZE; @@ -2336,7 +2411,7 @@ free_mem_section (GCMemSection *section) { char *data = section->data; size_t size = section->size; - DEBUG (2, fprintf (gc_debug_file, "Freed section %p, size %d\n", data, size)); + DEBUG (2, fprintf (gc_debug_file, "Freed section %p, size %zd\n", data, size)); free_os_memory (data, size); free_internal_mem (section); total_alloc -= size; @@ -2360,13 +2435,13 @@ minor_collect_or_expand_inner (size_t size) if (do_minor_collection) { stop_world (); collect_nursery (size); - DEBUG (2, fprintf (gc_debug_file, "Heap size: %d, LOS size: %d\n", total_alloc, los_memory_usage)); + DEBUG (2, fprintf (gc_debug_file, "Heap size: %zd, LOS size: %zd\n", total_alloc, los_memory_usage)); restart_world (); /* this also sets the proper pointers for the next allocation */ if (!search_fragment_for_size (size)) { int i; /* TypeBuilder and MonoMethod are killing mcs with fragmentation */ - DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %d alloc (%d pinned)", size, last_num_pinned)); + DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)", size, last_num_pinned)); for (i = 0; i < last_num_pinned; ++i) { DEBUG (3, fprintf (gc_debug_file, "Bastard pinning obj %p (%s), size: %d\n", pin_queue [i], safe_name (pin_queue [i]), safe_object_get_size (pin_queue [i]))); } @@ -2385,10 +2460,6 @@ minor_collect_or_expand_inner (size_t size) * Internal memory can be handled with a freelist for small objects. */ -#ifndef MAP_ANONYMOUS -#define MAP_ANONYMOUS MAP_ANON -#endif - /* * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes). * This must not require any lock. @@ -2397,21 +2468,12 @@ static void* get_os_memory (size_t size, int activate) { void *ptr; - unsigned long prot_flags = activate? PROT_READ|PROT_WRITE: PROT_NONE; + unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE; + prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON; size += pagesize - 1; size &= ~(pagesize - 1); - ptr = mmap (0, size, prot_flags, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (ptr == (void*)-1) { - int fd = open ("/dev/zero", O_RDONLY); - if (fd != -1) { - ptr = mmap (0, size, prot_flags, MAP_PRIVATE, fd, 0); - close (fd); - } - if (ptr == (void*)-1) { - return NULL; - } - } + ptr = mono_valloc (0, size, prot_flags); return ptr; } @@ -2579,11 +2641,11 @@ build_freelist (PinnedChunk *chunk, int slot, int size, char *start_page, char * void **p, **end; int count = 0; /*g_print ("building freelist for slot %d, size %d in %p\n", slot, size, chunk);*/ - p = start_page; + p = (void**)start_page; end = (void**)(end_page - size); g_assert (!chunk->free_list [slot]); chunk->free_list [slot] = p; - while ((char*)p + size <= end) { + while ((char*)p + size <= (char*)end) { count++; *p = (void*)((char*)p + size); p = *p; @@ -2623,7 +2685,7 @@ alloc_pinned_chunk (size_t size) /* allocate the first page to the freelist */ chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE; build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE)); - DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size)); + DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %zd\n", chunk, size)); min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data); max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size)); return chunk; @@ -2696,6 +2758,7 @@ static void* get_internal_mem (size_t size) { return calloc (1, size); +#if 0 int slot; void *res = NULL; PinnedChunk *pchunk; @@ -2719,13 +2782,14 @@ get_internal_mem (size_t size) internal_chunk_list = pchunk; res = get_chunk_freelist (pchunk, slot); return res; +#endif } static void free_internal_mem (void *addr) { free (addr); - return; +#if 0 PinnedChunk *pchunk; for (pchunk = internal_chunk_list; pchunk; pchunk = pchunk->next) { /*printf ("trying to free %p in %p (pages: %d)\n", addr, pchunk, pchunk->num_pages);*/ @@ -2741,6 +2805,7 @@ free_internal_mem (void *addr) } printf ("free of %p failed\n", addr); g_assert_not_reached (); +#endif } /* @@ -2759,7 +2824,7 @@ static void free_large_object (LOSObject *obj) { size_t size = obj->size; - DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %d\n", obj->data, obj->size)); + DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size)); los_memory_usage -= size; size += sizeof (LOSObject); @@ -2785,7 +2850,7 @@ alloc_large_inner (MonoVTable *vtable, size_t size) int just_did_major_gc = FALSE; if (los_memory_usage > next_los_collection) { - DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %d (los already: %u, limit: %u)\n", size, los_memory_usage, next_los_collection)); + DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection)); just_did_major_gc = TRUE; stop_world (); major_collection (); @@ -2808,7 +2873,7 @@ alloc_large_inner (MonoVTable *vtable, size_t size) los_object_list = obj; los_memory_usage += size; los_num_objects++; - DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %d\n", obj->data, vtable, vtable->klass->name, size)); + DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size)); return obj->data; } @@ -2820,7 +2885,7 @@ static gboolean search_fragment_for_size (size_t size) { Fragment *frag, *prev; - DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %d\n", nursery_frag_real_end, size)); + DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, size: %zd\n", nursery_frag_real_end, size)); prev = NULL; for (frag = nursery_fragments; frag; frag = frag->next) { if (size <= (frag->fragment_end - frag->fragment_start)) { @@ -2833,7 +2898,7 @@ search_fragment_for_size (size_t size) nursery_frag_real_end = frag->fragment_end; nursery_temp_end = MIN (nursery_frag_real_end, nursery_next + size + SCAN_START_SIZE); - DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %d (req: %d)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size)); + DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size)); frag->next = fragment_freelist; fragment_freelist = frag; return TRUE; @@ -2844,7 +2909,7 @@ search_fragment_for_size (size_t size) } /* - * size is already rounded up. + * size is already rounded up and we hold the GC lock. */ static void* alloc_degraded (MonoVTable *vtable, size_t size) @@ -2853,18 +2918,18 @@ alloc_degraded (MonoVTable *vtable, size_t size) void **p = NULL; for (section = section_list; section; section = section->next) { if (section != nursery_section && (section->end_data - section->next_data) >= size) { - p = section->next_data; + p = (void**)section->next_data; break; } } if (!p) { section = alloc_section (nursery_section->size * 4); /* FIXME: handle OOM */ - p = section->next_data; + p = (void**)section->next_data; } section->next_data += size; degraded_mode += size; - DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %d in section %p\n", p, vtable, vtable->klass->name, size, section)); + DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section)); *p = vtable; return p; } @@ -2935,14 +3000,14 @@ mono_gc_alloc_obj (MonoVTable *vtable, size_t size) } } else { /* record the scan start so we can find pinned objects more easily */ - nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = p; + nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p; /* we just bump nursery_temp_end as well */ nursery_temp_end = MIN (nursery_frag_real_end, nursery_next + SCAN_START_SIZE); DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", nursery_next, nursery_temp_end)); } } } - DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %d\n", p, vtable, vtable->klass->name, size)); + DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size)); *p = vtable; UNLOCK_GC; @@ -2970,7 +3035,7 @@ mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size) p = alloc_from_freelist (size); memset (p, 0, size); } - DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %d\n", p, vtable, vtable->klass->name, size)); + DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size)); *p = vtable; UNLOCK_GC; return p; @@ -2990,7 +3055,7 @@ mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size) #define object_is_fin_ready(obj) (!object_is_pinned (obj) && !object_is_forwarded (obj)) static void -finalize_in_range (void **start, void **end) +finalize_in_range (char *start, char *end) { FinalizeEntry *entry, *prev; int i; @@ -2999,7 +3064,7 @@ finalize_in_range (void **start, void **end) for (i = 0; i < finalizable_hash_size; ++i) { prev = NULL; for (entry = finalizable_hash [i]; entry;) { - if (entry->object >= start && entry->object < end) { + if ((char*)entry->object >= start && (char*)entry->object < end && ((char*)entry->object < to_space || (char*)entry->object >= to_space_end)) { if (object_is_fin_ready (entry->object)) { char *from; FinalizeEntry *next; @@ -3032,14 +3097,14 @@ finalize_in_range (void **start, void **end) } static void -null_link_in_range (void **start, void **end) +null_link_in_range (char *start, char *end) { FinalizeEntry *entry, *prev; int i; for (i = 0; i < disappearing_link_hash_size; ++i) { prev = NULL; for (entry = disappearing_link_hash [i]; entry;) { - if (entry->object >= start && entry->object < end) { + if ((char*)entry->object >= start && (char*)entry->object < end && ((char*)entry->object < to_space || (char*)entry->object >= to_space_end)) { if (object_is_fin_ready (entry->object)) { void **p = entry->data; FinalizeEntry *old; @@ -3053,6 +3118,7 @@ null_link_in_range (void **start, void **end) old = entry->next; free_internal_mem (entry); entry = old; + num_disappearing_links--; continue; } else { void **link; @@ -3408,29 +3474,18 @@ mono_gc_deregister_root (char* addr) * ###################################################################### */ -#undef pthread_create -#undef pthread_join -#undef pthread_detach - -typedef struct { - void *(*start_routine) (void *); - void *arg; - int flags; - sem_t registered; -} SgenThreadStartInfo; - /* eventually share with MonoThread? */ typedef struct _SgenThreadInfo SgenThreadInfo; struct _SgenThreadInfo { SgenThreadInfo *next; - pthread_t id; + ARCH_THREAD_TYPE id; unsigned int stop_count; /* to catch duplicate signals */ int signal; int skip; void *stack_end; void *stack_start; - RememberedSet **remset; + RememberedSet *remset; }; /* FIXME: handle large/small config */ @@ -3438,6 +3493,9 @@ struct _SgenThreadInfo { #define HASH_PTHREAD_T(id) (((unsigned int)(id) >> 4) * 2654435761u) static SgenThreadInfo* thread_table [THREAD_HASH_SIZE]; + +#if USE_SIGNAL_BASED_START_STOP_WORLD + static sem_t suspend_ack_semaphore; static unsigned int global_stop_count = 0; static int suspend_signal_num = SIGPWR; @@ -3447,13 +3505,13 @@ static mword cur_thread_regs [ARCH_NUM_REGS] = {0}; /* LOCKING: assumes the GC lock is held */ static SgenThreadInfo* -thread_info_lookup (pthread_t id) +thread_info_lookup (ARCH_THREAD_TYPE id) { unsigned int hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE; SgenThreadInfo *info; info = thread_table [hash]; - while (info && !pthread_equal (info->id, id)) { + while (info && !ARCH_THREAD_EQUALS (info->id, id)) { info = info->next; } return info; @@ -3463,7 +3521,7 @@ static void update_current_thread_stack (void *start) { void *ptr = cur_thread_regs; - SgenThreadInfo *info = thread_info_lookup (pthread_self ()); + SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ()); info->stack_start = align_pointer (&ptr); ARCH_STORE_REGS (ptr); } @@ -3490,7 +3548,7 @@ thread_handshake (int signum) for (i = 0; i < THREAD_HASH_SIZE; ++i) { for (info = thread_table [i]; info; info = info->next) { DEBUG (4, fprintf (gc_debug_file, "considering thread %p for signal %d (%s)\n", info, signum, signal_desc (signum))); - if (pthread_equal (info->id, me)) { + if (ARCH_THREAD_EQUALS (info->id, me)) { DEBUG (4, fprintf (gc_debug_file, "Skip (equal): %p, %p\n", (void*)me, (void*)info->id)); continue; } @@ -3581,7 +3639,7 @@ stop_world (void) int count; global_stop_count++; - DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (pthread_self ()), (gpointer)pthread_self ())); + DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, thread_info_lookup (ARCH_GET_THREAD ()), (gpointer)ARCH_GET_THREAD ())); TV_GETTIME (stop_world_time); count = thread_handshake (suspend_signal_num); DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count)); @@ -3604,6 +3662,8 @@ restart_world (void) return count; } +#endif /* USE_SIGNAL_BASED_START_STOP_WORLD */ + /* * Identify objects pinned in a thread stack and its registers. */ @@ -3616,15 +3676,15 @@ pin_thread_data (void *start_nursery, void *end_nursery) for (i = 0; i < THREAD_HASH_SIZE; ++i) { for (info = thread_table [i]; info; info = info->next) { if (info->skip) { - DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); + DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); continue; } - DEBUG (2, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); + DEBUG (2, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery); } } DEBUG (2, fprintf (gc_debug_file, "Scanning current thread registers\n")); - conservatively_pin_objects_from (cur_thread_regs, cur_thread_regs + ARCH_NUM_REGS, start_nursery, end_nursery); + conservatively_pin_objects_from ((void*)cur_thread_regs, (void*)(cur_thread_regs + ARCH_NUM_REGS), start_nursery, end_nursery); } static void @@ -3641,7 +3701,7 @@ find_pinning_ref_from_thread (char *obj, size_t size) continue; while (start < (char**)info->stack_end) { if (*start >= obj && *start < endobj) { - DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, info->id, start, info->stack_start, info->stack_end)); + DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)info->id, start, info->stack_start, info->stack_end)); } start++; } @@ -3671,7 +3731,7 @@ handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global switch ((*p) & REMSET_TYPE_MASK) { case REMSET_LOCATION: ptr = (void**)(*p); - if ((ptr < start_nursery || ptr >= end_nursery) && ptr_in_heap (ptr)) { + if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery) && ptr_in_heap (ptr)) { *ptr = copy_object (*ptr, start_nursery, end_nursery); DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr)); if (!global && *ptr >= start_nursery && *ptr < end_nursery) @@ -3682,7 +3742,7 @@ handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global return p + 1; case REMSET_RANGE: ptr = (void**)(*p & ~REMSET_TYPE_MASK); - if ((ptr >= start_nursery && ptr < end_nursery) || !ptr_in_heap (ptr)) + if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery) || !ptr_in_heap (ptr)) return p + 2; count = p [1]; while (count-- > 0) { @@ -3695,7 +3755,7 @@ handle_remset (mword *p, void *start_nursery, void *end_nursery, gboolean global return p + 2; case REMSET_OBJECT: ptr = (void**)(*p & ~REMSET_TYPE_MASK); - if ((ptr >= start_nursery && ptr < end_nursery) || !ptr_in_heap (ptr)) + if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery) || !ptr_in_heap (ptr)) return p + 1; scan_object (*ptr, start_nursery, end_nursery); return p + 1; @@ -3715,7 +3775,7 @@ scan_from_remsets (void *start_nursery, void *end_nursery) /* the global one */ for (remset = global_remset; remset; remset = remset->next) { - DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %d\n", remset->data, remset->store_next, remset->store_next - remset->data)); + DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data)); for (p = remset->data; p < remset->store_next;) { p = handle_remset (p, start_nursery, end_nursery, TRUE); } @@ -3724,7 +3784,7 @@ scan_from_remsets (void *start_nursery, void *end_nursery) for (i = 0; i < THREAD_HASH_SIZE; ++i) { for (info = thread_table [i]; info; info = info->next) { for (remset = info->remset; remset; remset = next) { - DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %d\n", info, remset->data, remset->store_next, remset->store_next - remset->data)); + DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data)); for (p = remset->data; p < remset->store_next;) { p = handle_remset (p, start_nursery, end_nursery, FALSE); } @@ -3786,7 +3846,7 @@ gc_register_current_thread (void *addr) SgenThreadInfo* info = malloc (sizeof (SgenThreadInfo)); if (!info) return NULL; - info->id = pthread_self (); + info->id = ARCH_GET_THREAD (); info->stop_count = -1; info->skip = 0; info->signal = 0; @@ -3801,6 +3861,7 @@ gc_register_current_thread (void *addr) pthread_getattr_np (pthread_self (), &attr); pthread_attr_getstack (&attr, &sstart, &size); info->stack_end = (char*)sstart + size; + pthread_attr_destroy (&attr); } #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP) info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ()); @@ -3820,6 +3881,7 @@ gc_register_current_thread (void *addr) thread_table [hash] = info; remembered_set = info->remset = alloc_remset (DEFAULT_REMSET_SIZE, info); + pthread_setspecific (remembered_set_key, remembered_set); DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) (hash: %d)\n", info, (gpointer)info->id, hash)); return info; } @@ -3830,13 +3892,14 @@ unregister_current_thread (void) int hash; SgenThreadInfo *prev = NULL; SgenThreadInfo *p; - pthread_t id = pthread_self (); + RememberedSet *rset; + ARCH_THREAD_TYPE id = ARCH_GET_THREAD (); hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE; p = thread_table [hash]; assert (p); DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)p->id)); - while (!pthread_equal (p->id, id)) { + while (!ARCH_THREAD_EQUALS (p->id, id)) { prev = p; p = p->next; } @@ -3845,10 +3908,49 @@ unregister_current_thread (void) } else { prev->next = p->next; } + rset = p->remset; /* FIXME: transfer remsets if any */ + while (rset) { + RememberedSet *next = rset->next; + free_internal_mem (rset); + rset = next; + } free (p); } +static void +unregister_thread (void *k) +{ + LOCK_GC; + unregister_current_thread (); + UNLOCK_GC; +} + +gboolean +mono_gc_register_thread (void *baseptr) +{ + SgenThreadInfo *info; + LOCK_GC; + info = thread_info_lookup (ARCH_GET_THREAD ()); + if (info == NULL) + info = gc_register_current_thread (baseptr); + UNLOCK_GC; + return info != NULL; +} + +#if USE_PTHREAD_INTERCEPT + +#undef pthread_create +#undef pthread_join +#undef pthread_detach + +typedef struct { + void *(*start_routine) (void *); + void *arg; + int flags; + sem_t registered; +} SgenThreadStartInfo; + static void* gc_start_thread (void *arg) { @@ -3863,9 +3965,12 @@ gc_start_thread (void *arg) UNLOCK_GC; sem_post (&(start_info->registered)); result = start_func (t_arg); + /* + * this is done by the pthread key dtor LOCK_GC; unregister_current_thread (); UNLOCK_GC; + */ return result; } @@ -3906,17 +4011,7 @@ mono_gc_pthread_detach (pthread_t thread) return pthread_detach (thread); } -gboolean -mono_gc_register_thread (void *baseptr) -{ - SgenThreadInfo *info; - LOCK_GC; - info = thread_info_lookup (pthread_self ()); - if (info == NULL) - info = gc_register_current_thread (baseptr); - UNLOCK_GC; - return info != NULL; -} +#endif /* USE_PTHREAD_INTERCEPT */ /* * ###################################################################### @@ -3946,7 +4041,7 @@ void mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value) { RememberedSet *rs; - if (field_ptr >= nursery_start && field_ptr < nursery_real_end) { + if ((char*)field_ptr >= nursery_start && (char*)field_ptr < nursery_real_end) { *(void**)field_ptr = value; return; } @@ -3960,7 +4055,7 @@ mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* val rs = alloc_remset (rs->end_set - rs->data, (void*)1); rs->next = remembered_set; remembered_set = rs; - thread_info_lookup (pthread_self())->remset = rs; + thread_info_lookup (ARCH_GET_THREAD ())->remset = rs; *(rs->store_next++) = (mword)field_ptr; *(void**)field_ptr = value; } @@ -3969,7 +4064,7 @@ void mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value) { RememberedSet *rs = remembered_set; - if (slot_ptr >= nursery_start && slot_ptr < nursery_real_end) { + if ((char*)slot_ptr >= nursery_start && (char*)slot_ptr < nursery_real_end) { *(void**)slot_ptr = value; return; } @@ -3982,7 +4077,7 @@ mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* va rs = alloc_remset (rs->end_set - rs->data, (void*)1); rs->next = remembered_set; remembered_set = rs; - thread_info_lookup (pthread_self())->remset = rs; + thread_info_lookup (ARCH_GET_THREAD ())->remset = rs; *(rs->store_next++) = (mword)slot_ptr; *(void**)slot_ptr = value; } @@ -3991,7 +4086,7 @@ void mono_gc_wbarrier_arrayref_copy (MonoArray *arr, gpointer slot_ptr, int count) { RememberedSet *rs = remembered_set; - if (slot_ptr >= nursery_start && slot_ptr < nursery_real_end) + if ((char*)slot_ptr >= nursery_start && (char*)slot_ptr < nursery_real_end) return; DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p, %d\n", slot_ptr, count)); if (rs->store_next + 1 < rs->end_set) { @@ -4002,7 +4097,7 @@ mono_gc_wbarrier_arrayref_copy (MonoArray *arr, gpointer slot_ptr, int count) rs = alloc_remset (rs->end_set - rs->data, (void*)1); rs->next = remembered_set; remembered_set = rs; - thread_info_lookup (pthread_self())->remset = rs; + thread_info_lookup (ARCH_GET_THREAD ())->remset = rs; *(rs->store_next++) = (mword)slot_ptr | REMSET_RANGE; *(rs->store_next++) = count; } @@ -4011,7 +4106,7 @@ void mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value) { RememberedSet *rs = remembered_set; - if (ptr >= nursery_start && ptr < nursery_real_end) { + if ((char*)ptr >= nursery_start && (char*)ptr < nursery_real_end) { DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr)); *(void**)ptr = value; return; @@ -4026,7 +4121,7 @@ mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value) rs = alloc_remset (rs->end_set - rs->data, (void*)1); rs->next = remembered_set; remembered_set = rs; - thread_info_lookup (pthread_self())->remset = rs; + thread_info_lookup (ARCH_GET_THREAD ())->remset = rs; *(rs->store_next++) = (mword)ptr; *(void**)ptr = value; } @@ -4034,7 +4129,7 @@ mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value) void mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass) { - if (dest >= nursery_start && dest < nursery_real_end) { + if ((char*)dest >= nursery_start && (char*)dest < nursery_real_end) { return; } DEBUG (1, fprintf (gc_debug_file, "Adding value remset at %p, count %d for class %s\n", dest, count, klass->name)); @@ -4057,10 +4152,218 @@ mono_gc_wbarrier_object (MonoObject* obj) rs = alloc_remset (rs->end_set - rs->data, (void*)1); rs->next = remembered_set; remembered_set = rs; - thread_info_lookup (pthread_self())->remset = rs; + thread_info_lookup (ARCH_GET_THREAD ())->remset = rs; *(rs->store_next++) = (mword)obj | REMSET_OBJECT; } +/* + * ###################################################################### + * ######## Collector debugging + * ###################################################################### + */ + +static mword* +find_in_remset_loc (mword *p, char *addr, gboolean *found) +{ + void **ptr; + mword count; + + switch ((*p) & REMSET_TYPE_MASK) { + case REMSET_LOCATION: + if (*p == (mword)addr) + *found = TRUE; + return p + 1; + case REMSET_RANGE: + ptr = (void**)(*p & ~REMSET_TYPE_MASK); + count = p [1]; + if ((void**)addr >= ptr && (void**)addr < ptr + count) + *found = TRUE; + return p + 2; + case REMSET_OBJECT: + ptr = (void**)(*p & ~REMSET_TYPE_MASK); + count = safe_object_get_size ((MonoObject*)ptr); + count += (ALLOC_ALIGN - 1); + count &= (ALLOC_ALIGN - 1); + count /= sizeof (mword); + if ((void**)addr >= ptr && (void**)addr < ptr + count) + *found = TRUE; + return p + 1; + default: + g_assert_not_reached (); + } + return NULL; +} + +/* + * Return whenever ADDR occurs in the remembered sets + */ +static gboolean +find_in_remsets (char *addr) +{ + int i; + SgenThreadInfo *info; + RememberedSet *remset; + mword *p; + gboolean found = FALSE; + + /* the global one */ + for (remset = global_remset; remset; remset = remset->next) { + DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data)); + for (p = remset->data; p < remset->store_next;) { + p = find_in_remset_loc (p, addr, &found); + if (found) + return TRUE; + } + } + /* the per-thread ones */ + for (i = 0; i < THREAD_HASH_SIZE; ++i) { + for (info = thread_table [i]; info; info = info->next) { + for (remset = info->remset; remset; remset = remset->next) { + DEBUG (4, fprintf (gc_debug_file, "Scanning remset for thread %p, range: %p-%p, size: %zd\n", info, remset->data, remset->store_next, remset->store_next - remset->data)); + for (p = remset->data; p < remset->store_next;) { + p = find_in_remset_loc (p, addr, &found); + if (found) + return TRUE; + } + } + } + } + + return FALSE; +} + +#undef HANDLE_PTR +#define HANDLE_PTR(ptr,obj) do { \ + if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \ + if (!find_in_remsets ((char*)(ptr))) { \ + fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \ + g_assert_not_reached (); \ + } \ + } \ + } while (0) + +/* + * Check that each object reference inside the area which points into the nursery + * can be found in the remembered sets. + */ +static void __attribute__((noinline)) +check_remsets_for_area (char *start, char *end) +{ + GCVTable *vt; + size_t skip_size; + int type; + int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0; + new_obj_references = 0; + obj_references_checked = 0; + while (start < end) { + if (!*(void**)start) { + start += sizeof (void*); /* should be ALLOC_ALIGN, really */ + continue; + } + vt = (GCVTable*)LOAD_VTABLE (start); + DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name)); + if (0) { + MonoObject *obj = (MonoObject*)start; + g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name); + } + type = vt->desc & 0x7; + if (type == DESC_TYPE_STRING) { + STRING_SIZE (skip_size, start); + start += skip_size; + type_str++; + continue; + } else if (type == DESC_TYPE_RUN_LENGTH) { + OBJ_RUN_LEN_SIZE (skip_size, vt, start); + g_assert (skip_size); + OBJ_RUN_LEN_FOREACH_PTR (vt,start); + start += skip_size; + type_rlen++; + continue; + } else if (type == DESC_TYPE_VECTOR) { // includes ARRAY, too + skip_size = (vt->desc >> LOW_TYPE_BITS) & MAX_ELEMENT_SIZE; + skip_size *= mono_array_length ((MonoArray*)start); + skip_size += sizeof (MonoArray); + skip_size += (ALLOC_ALIGN - 1); + skip_size &= ~(ALLOC_ALIGN - 1); + OBJ_VECTOR_FOREACH_PTR (vt, start); + if (((MonoArray*)start)->bounds) { + /* account for the bounds */ + skip_size += sizeof (MonoArrayBounds) * vt->klass->rank; + } + start += skip_size; + type_vector++; + continue; + } else if (type == DESC_TYPE_SMALL_BITMAP) { + OBJ_BITMAP_SIZE (skip_size, vt, start); + g_assert (skip_size); + OBJ_BITMAP_FOREACH_PTR (vt,start); + start += skip_size; + type_bitmap++; + continue; + } else if (type == DESC_TYPE_LARGE_BITMAP) { + skip_size = safe_object_get_size ((MonoObject*)start); + skip_size += (ALLOC_ALIGN - 1); + skip_size &= ~(ALLOC_ALIGN - 1); + OBJ_LARGE_BITMAP_FOREACH_PTR (vt,start); + start += skip_size; + type_lbit++; + continue; + } else if (type == DESC_TYPE_COMPLEX) { + /* this is a complex object */ + skip_size = safe_object_get_size ((MonoObject*)start); + skip_size += (ALLOC_ALIGN - 1); + skip_size &= ~(ALLOC_ALIGN - 1); + OBJ_COMPLEX_FOREACH_PTR (vt, start); + start += skip_size; + type_complex++; + continue; + } else if (type == DESC_TYPE_COMPLEX_ARR) { + /* this is an array of complex structs */ + skip_size = mono_array_element_size (((MonoVTable*)vt)->klass); + skip_size *= mono_array_length ((MonoArray*)start); + skip_size += sizeof (MonoArray); + skip_size += (ALLOC_ALIGN - 1); + skip_size &= ~(ALLOC_ALIGN - 1); + OBJ_COMPLEX_ARR_FOREACH_PTR (vt, start); + if (((MonoArray*)start)->bounds) { + /* account for the bounds */ + skip_size += sizeof (MonoArrayBounds) * vt->klass->rank; + } + start += skip_size; + type_complex++; + continue; + } else { + g_assert (0); + } + } +} + +/* + * Perform consistency check of the heap. + * + * Assumes the world is stopped. + */ +void +check_consistency (void) +{ + GCMemSection *section; + + // Need to add more checks + // FIXME: Create a general heap enumeration function and use that + + DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n")); + + // Check that oldspace->newspace pointers are registered with the collector + for (section = section_list; section; section = section->next) { + if (section->role == MEMORY_ROLE_GEN0) + continue; + DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data))); + check_remsets_for_area (section->data, section->next_data); + } + + DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n")); +} + /* * ###################################################################### * ######## Other mono public interface functions. @@ -4096,13 +4399,6 @@ mono_gc_collection_count (int generation) return num_major_gcs; } -int -mono_gc_get_generation (MonoObject *object) -{ - /* FIXME */ - return 0; -} - gint64 mono_gc_get_used_size (void) { @@ -4218,7 +4514,7 @@ mono_gc_is_gc_thread (void) { gboolean result; LOCK_GC; - result = thread_info_lookup (pthread_self ()) != NULL; + result = thread_info_lookup (ARCH_GET_THREAD ()) != NULL; UNLOCK_GC; return result; } @@ -4229,12 +4525,13 @@ mono_gc_base_init (void) char *env; struct sigaction sinfo; + LOCK_INIT (gc_mutex); LOCK_GC; if (gc_initialized) { UNLOCK_GC; return; } - gc_initialized = TRUE; + pagesize = mono_pagesize (); gc_debug_file = stderr; /* format: MONO_GC_DEBUG=l[,filename] where l is a debug level 0-9 */ if ((env = getenv ("MONO_GC_DEBUG"))) { @@ -4273,9 +4570,29 @@ mono_gc_base_init (void) global_remset = alloc_remset (1024, NULL); global_remset->next = NULL; + pthread_key_create (&remembered_set_key, unregister_thread); + gc_initialized = TRUE; UNLOCK_GC; mono_gc_register_thread (&sinfo); } +MonoMethod* +mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box) +{ + return NULL; +} + +int +mono_gc_get_managed_allocator_type (MonoMethod *managed_alloc) +{ + return -1; +} + +MonoMethod* +mono_gc_get_managed_allocator_by_type (int atype) +{ + return NULL; +} + #endif /* HAVE_SGEN_GC */