From 779d417c8f950d9ca8bc25d857ae24af9542ca0e Mon Sep 17 00:00:00 2001 From: Vlad Brezae Date: Fri, 10 Feb 2017 21:02:35 +0200 Subject: [PATCH] [sgen] Generational mono g hashtable We mark cards when storing in hashtables. During minors, instead of scanning the entire tables, we only scan the sections for the dirty cards. In order to simplify the implementation we split into separate key/value tables. To achieve this, instead of using an user descriptor for the hashtable structure, we directly register the tables and we create a new type of root descriptor for it to teach sgen how to find pointers in it. In addition to the precise scan of the root, which is done during majors, we also support a remset scan for it. We expect all roots registered as ROOT_TYPE_WBARRIER, to use this new descriptor (ROOT_DESC_VECTOR). In order for a root descriptor to be in the ROOT_TYPE_WBARRIER category, it has to have a defined remset scan mode. --- mono/metadata/boehm-gc.c | 6 + mono/metadata/mono-hash.c | 187 ++++++++++---------- mono/metadata/null-gc.c | 6 + mono/metadata/sgen-mono.c | 9 + mono/sgen/gc-internal-agnostic.h | 3 + mono/sgen/sgen-cardtable.c | 4 + mono/sgen/sgen-debug.c | 18 ++ mono/sgen/sgen-descriptor.c | 6 + mono/sgen/sgen-descriptor.h | 1 + mono/sgen/sgen-gc.c | 115 +++++++++++- mono/sgen/sgen-gc.h | 3 + mono/sgen/sgen-marksweep-drain-gray-stack.h | 12 +- mono/sgen/sgen-marksweep.c | 1 + 13 files changed, 272 insertions(+), 99 deletions(-) diff --git a/mono/metadata/boehm-gc.c b/mono/metadata/boehm-gc.c index dcfd310f649..f0fa3579f72 100644 --- a/mono/metadata/boehm-gc.c +++ b/mono/metadata/boehm-gc.c @@ -617,6 +617,12 @@ mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits) return (gpointer)GC_make_descriptor ((GC_bitmap)bitmap, numbits); } +void* +mono_gc_make_vector_descr (void) +{ + return NULL; +} + void* mono_gc_make_root_descr_all_refs (int numbits) { diff --git a/mono/metadata/mono-hash.c b/mono/metadata/mono-hash.c index 5dbb7b89569..6ca6801892d 100644 --- a/mono/metadata/mono-hash.c +++ b/mono/metadata/mono-hash.c @@ -44,18 +44,12 @@ #define mg_free(x) g_free(x) #endif -typedef struct _Slot Slot; - -struct _Slot { - MonoObject *key; - MonoObject *value; -}; - struct _MonoGHashTable { GHashFunc hash_func; GEqualFunc key_equal_func; - Slot *table; + MonoObject **keys; + MonoObject **values; int table_size; int in_use; GDestroyNotify value_destroy_func, key_destroy_func; @@ -64,12 +58,6 @@ struct _MonoGHashTable { const char *msg; }; -#ifdef HAVE_SGEN_GC -static MonoGCDescriptor table_hash_descr = MONO_GC_DESCRIPTOR_NULL; - -static void mono_g_hash_mark (void *addr, MonoGCMarkFunc mark_func, void *gc_data); -#endif - #if UNUSED static gboolean test_prime (int x) @@ -105,13 +93,31 @@ calc_prime (int x) /* We triple the table size at rehash time, similar with previous implementation */ #define HASH_TABLE_RESIZE_RATIO 3 +static inline void mono_g_hash_table_key_store (MonoGHashTable *hash, int slot, MonoObject* key) +{ + MonoObject **key_addr = &hash->keys [slot]; + if (hash->gc_type & MONO_HASH_KEY_GC) + mono_gc_wbarrier_generic_store (key_addr, key); + else + *key_addr = key; +} + +static inline void mono_g_hash_table_value_store (MonoGHashTable *hash, int slot, MonoObject* value) +{ + MonoObject **value_addr = &hash->values [slot]; + if (hash->gc_type & MONO_HASH_VALUE_GC) + mono_gc_wbarrier_generic_store (value_addr, value); + else + *value_addr = value; +} + /* Returns position of key or of an empty slot for it */ static inline int mono_g_hash_table_find_slot (MonoGHashTable *hash, const MonoObject *key) { guint i = ((*hash->hash_func) (key)) % hash->table_size; GEqualFunc equal = hash->key_equal_func; - while (hash->table [i].key && !(*equal) (hash->table [i].key, key)) { + while (hash->keys [i] && !(*equal) (hash->keys [i], key)) { i++; if (i == hash->table_size) i = 0; @@ -140,7 +146,8 @@ mono_g_hash_table_new_type (GHashFunc hash_func, GEqualFunc key_equal_func, Mono hash->key_equal_func = key_equal_func; hash->table_size = g_spaced_primes_closest (1); - hash->table = mg_new0 (Slot, hash->table_size); + hash->keys = mg_new0 (MonoObject*, hash->table_size); + hash->values = mg_new0 (MonoObject*, hash->table_size); hash->gc_type = type; hash->source = source; @@ -150,13 +157,10 @@ mono_g_hash_table_new_type (GHashFunc hash_func, GEqualFunc key_equal_func, Mono g_error ("wrong type for gc hashtable"); #ifdef HAVE_SGEN_GC - /* - * We use a user defined marking function to avoid having to register a GC root for - * each hash node. - */ - if (!table_hash_descr) - table_hash_descr = mono_gc_make_root_descr_user (mono_g_hash_mark); - mono_gc_register_root_wbarrier ((char*)hash, sizeof (MonoGHashTable), table_hash_descr, source, msg); + if (hash->gc_type & MONO_HASH_KEY_GC) + mono_gc_register_root_wbarrier ((char*)hash->keys, sizeof (MonoObject*) * hash->table_size, mono_gc_make_vector_descr (), hash->source, hash->msg); + if (hash->gc_type & MONO_HASH_VALUE_GC) + mono_gc_register_root_wbarrier ((char*)hash->values, sizeof (MonoObject*) * hash->table_size, mono_gc_make_vector_descr (), hash->source, hash->msg); #endif return hash; @@ -165,7 +169,8 @@ mono_g_hash_table_new_type (GHashFunc hash_func, GEqualFunc key_equal_func, Mono typedef struct { MonoGHashTable *hash; int new_size; - Slot *table; + MonoObject **keys; + MonoObject **values; } RehashData; static void* @@ -174,21 +179,24 @@ do_rehash (void *_data) RehashData *data = (RehashData *)_data; MonoGHashTable *hash = data->hash; int current_size, i; - Slot *old_table; + MonoObject **old_keys; + MonoObject **old_values; current_size = hash->table_size; hash->table_size = data->new_size; - old_table = hash->table; - hash->table = data->table; + old_keys = hash->keys; + old_values = hash->values; + hash->keys = data->keys; + hash->values = data->values; for (i = 0; i < current_size; i++) { - if (old_table [i].key) { - int slot = mono_g_hash_table_find_slot (hash, old_table [i].key); - hash->table [slot].key = old_table [i].key; - hash->table [slot].value = old_table [i].value; + if (old_keys [i]) { + int slot = mono_g_hash_table_find_slot (hash, old_keys [i]); + mono_g_hash_table_key_store (hash, slot, old_keys [i]); + mono_g_hash_table_value_store (hash, slot, old_values [i]); } } - return old_table; + return NULL; } static void @@ -197,7 +205,8 @@ rehash (MonoGHashTable *hash) MONO_REQ_GC_UNSAFE_MODE; //we must run in unsafe mode to make rehash safe RehashData data; - void *old_table G_GNUC_UNUSED; /* unused on Boehm */ + void *old_keys G_GNUC_UNUSED = hash->keys; /* unused on Boehm */ + void *old_values G_GNUC_UNUSED = hash->values; /* unused on Boehm */ data.hash = hash; /* @@ -205,16 +214,31 @@ rehash (MonoGHashTable *hash) * to allow also for compaction. */ data.new_size = g_spaced_primes_closest (hash->in_use / HASH_TABLE_MAX_LOAD_FACTOR * HASH_TABLE_RESIZE_RATIO); - data.table = mg_new0 (Slot, data.new_size); + data.keys = mg_new0 (MonoObject*, data.new_size); + data.values = mg_new0 (MonoObject*, data.new_size); + +#ifdef HAVE_SGEN_GC + if (hash->gc_type & MONO_HASH_KEY_GC) + mono_gc_register_root_wbarrier ((char*)data.keys, sizeof (MonoObject*) * data.new_size, mono_gc_make_vector_descr (), hash->source, hash->msg); + if (hash->gc_type & MONO_HASH_VALUE_GC) + mono_gc_register_root_wbarrier ((char*)data.values, sizeof (MonoObject*) * data.new_size, mono_gc_make_vector_descr (), hash->source, hash->msg); +#endif if (!mono_threads_is_coop_enabled ()) { - old_table = mono_gc_invoke_with_gc_lock (do_rehash, &data); + mono_gc_invoke_with_gc_lock (do_rehash, &data); } else { /* We cannot be preempted */ - old_table = do_rehash (&data); + do_rehash (&data); } - mg_free (old_table); +#ifdef HAVE_SGEN_GC + if (hash->gc_type & MONO_HASH_KEY_GC) + mono_gc_deregister_root ((char*)old_keys); + if (hash->gc_type & MONO_HASH_VALUE_GC) + mono_gc_deregister_root ((char*)old_values); +#endif + mg_free (old_keys); + mg_free (old_values); } guint @@ -245,9 +269,9 @@ mono_g_hash_table_lookup_extended (MonoGHashTable *hash, gconstpointer key, gpoi slot = mono_g_hash_table_find_slot (hash, key); - if (hash->table [slot].key) { - *orig_key = hash->table [slot].key; - *value = hash->table [slot].value; + if (hash->keys [slot]) { + *orig_key = hash->keys [slot]; + *value = hash->values [slot]; return TRUE; } @@ -263,8 +287,8 @@ mono_g_hash_table_foreach (MonoGHashTable *hash, GHFunc func, gpointer user_data g_return_if_fail (func != NULL); for (i = 0; i < hash->table_size; i++) { - if (hash->table [i].key) - (*func)(hash->table [i].key, hash->table [i].value, user_data); + if (hash->keys [i]) + (*func)(hash->keys [i], hash->values [i], user_data); } } @@ -277,8 +301,8 @@ mono_g_hash_table_find (MonoGHashTable *hash, GHRFunc predicate, gpointer user_d g_return_val_if_fail (predicate != NULL, NULL); for (i = 0; i < hash->table_size; i++) { - if (hash->table [i].key && (*predicate)(hash->table [i].key, hash->table [i].value, user_data)) - return hash->table [i].value; + if (hash->keys [i] && (*predicate)(hash->keys [i], hash->values [i], user_data)) + return hash->values [i]; } return NULL; } @@ -291,15 +315,15 @@ mono_g_hash_table_remove (MonoGHashTable *hash, gconstpointer key) g_return_val_if_fail (hash != NULL, FALSE); slot = mono_g_hash_table_find_slot (hash, key); - if (!hash->table [slot].key) + if (!hash->keys [slot]) return FALSE; if (hash->key_destroy_func) - (*hash->key_destroy_func)(hash->table [slot].key); - hash->table [slot].key = NULL; + (*hash->key_destroy_func)(hash->keys [slot]); + hash->keys [slot] = NULL; if (hash->value_destroy_func) - (*hash->value_destroy_func)(hash->table [slot].value); - hash->table [slot].value = NULL; + (*hash->value_destroy_func)(hash->values [slot]); + hash->values [slot] = NULL; hash->in_use--; /* @@ -314,18 +338,18 @@ mono_g_hash_table_remove (MonoGHashTable *hash, gconstpointer key) */ last_clear_slot = slot; slot = (slot + 1) % hash->table_size; - while (hash->table [slot].key) { - guint hashcode = ((*hash->hash_func)(hash->table [slot].key)) % hash->table_size; + while (hash->keys [slot]) { + guint hashcode = ((*hash->hash_func)(hash->keys [slot])) % hash->table_size; /* * We try to move the current element to last_clear_slot, but only if * it brings it closer to its normal position (hashcode) */ if ((last_clear_slot < slot && (hashcode > slot || hashcode <= last_clear_slot)) || (last_clear_slot > slot && (hashcode > slot && hashcode <= last_clear_slot))) { - hash->table [last_clear_slot].key = hash->table [slot].key; - hash->table [last_clear_slot].value = hash->table [slot].value; - hash->table [slot].key = NULL; - hash->table [slot].value = NULL; + mono_g_hash_table_key_store (hash, last_clear_slot, hash->keys [slot]); + mono_g_hash_table_value_store (hash, last_clear_slot, hash->values [slot]); + hash->keys [slot] = NULL; + hash->values [slot] = NULL; last_clear_slot = slot; } slot++; @@ -345,8 +369,8 @@ mono_g_hash_table_foreach_remove (MonoGHashTable *hash, GHRFunc func, gpointer u g_return_val_if_fail (func != NULL, 0); for (i = 0; i < hash->table_size; i++) { - if (hash->table [i].key && (*func)(hash->table [i].key, hash->table [i].value, user_data)) { - mono_g_hash_table_remove (hash, hash->table [i].key); + if (hash->keys [i] && (*func)(hash->keys [i], hash->values [i], user_data)) { + mono_g_hash_table_remove (hash, hash->keys [i]); count++; /* Retry current slot in case the removal shifted elements */ i--; @@ -365,18 +389,22 @@ mono_g_hash_table_destroy (MonoGHashTable *hash) g_return_if_fail (hash != NULL); #ifdef HAVE_SGEN_GC - mono_gc_deregister_root ((char*)hash); + if (hash->gc_type & MONO_HASH_KEY_GC) + mono_gc_deregister_root ((char*)hash->keys); + if (hash->gc_type & MONO_HASH_VALUE_GC) + mono_gc_deregister_root ((char*)hash->values); #endif for (i = 0; i < hash->table_size; i++) { - if (hash->table [i].key != NULL) { + if (hash->keys [i]) { if (hash->key_destroy_func) - (*hash->key_destroy_func)(hash->table [i].key); + (*hash->key_destroy_func)(hash->keys [i]); if (hash->value_destroy_func) - (*hash->value_destroy_func)(hash->table [i].value); + (*hash->value_destroy_func)(hash->values [i]); } } - mg_free (hash->table); + mg_free (hash->keys); + mg_free (hash->values); #ifdef HAVE_SGEN_GC mg_free (hash); #else @@ -395,18 +423,18 @@ mono_g_hash_table_insert_replace (MonoGHashTable *hash, gpointer key, gpointer v slot = mono_g_hash_table_find_slot (hash, key); - if (hash->table [slot].key) { + if (hash->keys [slot]) { if (replace) { if (hash->key_destroy_func) - (*hash->key_destroy_func)(hash->table [slot].key); - hash->table [slot].key = (MonoObject *)key; + (*hash->key_destroy_func)(hash->keys [slot]); + mono_g_hash_table_key_store (hash, slot, (MonoObject*)key); } if (hash->value_destroy_func) - (*hash->value_destroy_func) (hash->table [slot].value); - hash->table [slot].value = (MonoObject *)value; + (*hash->value_destroy_func) (hash->values [slot]); + mono_g_hash_table_value_store (hash, slot, (MonoObject*)value); } else { - hash->table [slot].key = (MonoObject *)key; - hash->table [slot].value = (MonoObject *)value; + mono_g_hash_table_key_store (hash, slot, (MonoObject*)key); + mono_g_hash_table_value_store (hash, slot, (MonoObject*)value); hash->in_use++; } } @@ -430,7 +458,7 @@ mono_g_hash_table_print_stats (MonoGHashTable *hash) gboolean wrapped_around = FALSE; while (TRUE) { - if (hash->table [i].key) { + if (hash->keys [i]) { chain_size++; } else { max_chain_size = MAX(max_chain_size, chain_size); @@ -449,22 +477,3 @@ mono_g_hash_table_print_stats (MonoGHashTable *hash) /* Rehash to a size that can fit the current elements */ printf ("Size: %d Table Size: %d Max Chain Length: %d\n", hash->in_use, hash->table_size, max_chain_size); } - -#ifdef HAVE_SGEN_GC - -/* GC marker function */ -static void -mono_g_hash_mark (void *addr, MonoGCMarkFunc mark_func, void *gc_data) -{ - MonoGHashTable *hash = (MonoGHashTable*)addr; - int i; - - for (i = 0; i < hash->table_size; i++) { - if (hash->gc_type & MONO_HASH_KEY_GC && hash->table [i].key) - mark_func (&hash->table [i].key, gc_data); - if (hash->gc_type & MONO_HASH_VALUE_GC && hash->table [i].value) - mark_func (&hash->table [i].value, gc_data); - } -} - -#endif diff --git a/mono/metadata/null-gc.c b/mono/metadata/null-gc.c index 9ea87bd844c..4ee7dd7c9c4 100644 --- a/mono/metadata/null-gc.c +++ b/mono/metadata/null-gc.c @@ -164,6 +164,12 @@ mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits) return NULL; } +void* +mono_gc_make_vector_descr (void) +{ + return NULL; +} + void* mono_gc_make_root_descr_all_refs (int numbits) { diff --git a/mono/metadata/sgen-mono.c b/mono/metadata/sgen-mono.c index beaf1b2f9fc..ab1c872fe07 100644 --- a/mono/metadata/sgen-mono.c +++ b/mono/metadata/sgen-mono.c @@ -1939,6 +1939,15 @@ precisely_report_roots_from (GCRootReport *report, void** start_root, void** end } break; } + case ROOT_DESC_VECTOR: { + void **p; + + for (p = start_root; p < end_root; p++) { + if (*p) + add_profile_gc_root (report, *p, MONO_PROFILE_GC_ROOT_OTHER, 0); + } + break; + } case ROOT_DESC_USER: { MonoGCRootMarkFunc marker = (MonoGCRootMarkFunc)sgen_get_user_descriptor_func (desc); root_report = report; diff --git a/mono/sgen/gc-internal-agnostic.h b/mono/sgen/gc-internal-agnostic.h index 62039f72679..dc9b091bf6f 100644 --- a/mono/sgen/gc-internal-agnostic.h +++ b/mono/sgen/gc-internal-agnostic.h @@ -93,6 +93,9 @@ MonoGCDescriptor mono_gc_make_descr_for_array (int vector, gsize *elem_bitmap, i /* simple interface for data structures needed in the runtime */ MonoGCDescriptor mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits); +/* Return a root descriptor for a vector with repeating refs bitmap */ +MonoGCDescriptor mono_gc_make_vector_descr (void); + /* Return a root descriptor for a root with all refs */ MonoGCDescriptor mono_gc_make_root_descr_all_refs (int numbits); diff --git a/mono/sgen/sgen-cardtable.c b/mono/sgen/sgen-cardtable.c index 56c54c87406..cb6c2c52a5f 100644 --- a/mono/sgen/sgen-cardtable.c +++ b/mono/sgen/sgen-cardtable.c @@ -412,6 +412,7 @@ sgen_card_table_clear_cards (void) /*XXX we could do this in 2 ways. using mincore or iterating over all sections/los objects */ sgen_major_collector_iterate_block_ranges (clear_cards); sgen_los_iterate_live_block_ranges (clear_cards); + sgen_wbroots_iterate_live_block_ranges (clear_cards); } static void @@ -433,6 +434,7 @@ sgen_card_table_scan_remsets (ScanCopyContext ctx) /*First we copy*/ sgen_major_collector_iterate_block_ranges (move_cards_to_shadow_table); sgen_los_iterate_live_block_ranges (move_cards_to_shadow_table); + sgen_wbroots_iterate_live_block_ranges (move_cards_to_shadow_table); /*Then we clear*/ sgen_card_table_clear_cards (); @@ -446,6 +448,8 @@ sgen_card_table_scan_remsets (ScanCopyContext ctx) SGEN_TV_GETTIME (atv); last_los_scan_time = SGEN_TV_ELAPSED (btv, atv); los_card_scan_time += last_los_scan_time; + + sgen_wbroots_scan_card_table (ctx); } guint8* diff --git a/mono/sgen/sgen-debug.c b/mono/sgen/sgen-debug.c index 195894f8cbf..a78fb3dad57 100644 --- a/mono/sgen/sgen-debug.c +++ b/mono/sgen/sgen-debug.c @@ -804,6 +804,15 @@ scan_roots_for_specific_ref (GCObject *key, int root_type) } break; } + case ROOT_DESC_VECTOR: { + void **p; + + for (p = start_root; p < (void**)root->end_root; p++) { + if (*p) + check_root_obj_specific_ref (root, key, (GCObject *)*p); + } + break; + } case ROOT_DESC_USER: { SgenUserRootMarkFunc marker = sgen_get_user_descriptor_func (desc); marker (start_root, check_root_obj_specific_ref_from_marker, NULL); @@ -908,6 +917,15 @@ sgen_scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type) } break; } + case ROOT_DESC_VECTOR: { + void **p; + + for (p = start_root; p < (void**)root->end_root; p++) { + if (*p) + check_obj_not_in_domain ((MonoObject **)*p); + } + break; + } case ROOT_DESC_USER: { SgenUserRootMarkFunc marker = sgen_get_user_descriptor_func (desc); marker (start_root, check_obj_not_in_domain_callback, NULL); diff --git a/mono/sgen/sgen-descriptor.c b/mono/sgen/sgen-descriptor.c index 248b07b8f50..f358c9da429 100644 --- a/mono/sgen/sgen-descriptor.c +++ b/mono/sgen/sgen-descriptor.c @@ -275,6 +275,12 @@ mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits) } } +SgenDescriptor +mono_gc_make_vector_descr (void) +{ + return MAKE_ROOT_DESC (ROOT_DESC_VECTOR, 0); +} + SgenDescriptor mono_gc_make_root_descr_all_refs (int numbits) { diff --git a/mono/sgen/sgen-descriptor.h b/mono/sgen/sgen-descriptor.h index 54e1b7681fb..e072ca45d32 100644 --- a/mono/sgen/sgen-descriptor.h +++ b/mono/sgen/sgen-descriptor.h @@ -114,6 +114,7 @@ enum { ROOT_DESC_BITMAP, ROOT_DESC_RUN_LEN, ROOT_DESC_COMPLEX, + ROOT_DESC_VECTOR, ROOT_DESC_USER, ROOT_DESC_TYPE_MASK = 0x7, ROOT_DESC_TYPE_SHIFT = 3, diff --git a/mono/sgen/sgen-gc.c b/mono/sgen/sgen-gc.c index 7c02172c6ee..ec27de4224b 100644 --- a/mono/sgen/sgen-gc.c +++ b/mono/sgen/sgen-gc.c @@ -858,6 +858,7 @@ static void precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, SgenDescriptor desc, ScanCopyContext ctx) { CopyOrMarkObjectFunc copy_func = ctx.ops->copy_or_mark_object; + ScanPtrFieldFunc scan_field_func = ctx.ops->scan_ptr_field; SgenGrayQueue *queue = ctx.queue; switch (desc & ROOT_DESC_TYPE_MASK) { @@ -892,6 +893,15 @@ precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, } break; } + case ROOT_DESC_VECTOR: { + void **p; + + for (p = start_root; p < end_root; p++) { + if (*p) + scan_field_func (NULL, (GCObject**)p, queue); + } + break; + } case ROOT_DESC_USER: { SgenUserRootMarkFunc marker = sgen_get_user_descriptor_func (desc); marker (start_root, single_arg_user_copy_or_mark, &ctx); @@ -1481,13 +1491,16 @@ enqueue_scan_from_roots_jobs (SgenGrayQueue *gc_thread_gray_queue, char *heap_st scrrj->root_type = ROOT_TYPE_NORMAL; sgen_workers_enqueue_job (&scrrj->scan_job.job, enqueue); - scrrj = (ScanFromRegisteredRootsJob*)sgen_thread_pool_job_alloc ("scan from registered roots wbarrier", job_scan_from_registered_roots, sizeof (ScanFromRegisteredRootsJob)); - scrrj->scan_job.ops = ops; - scrrj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue; - scrrj->heap_start = heap_start; - scrrj->heap_end = heap_end; - scrrj->root_type = ROOT_TYPE_WBARRIER; - sgen_workers_enqueue_job (&scrrj->scan_job.job, enqueue); + if (current_collection_generation == GENERATION_OLD) { + /* During minors we scan the cardtable for these roots instead */ + scrrj = (ScanFromRegisteredRootsJob*)sgen_thread_pool_job_alloc ("scan from registered roots wbarrier", job_scan_from_registered_roots, sizeof (ScanFromRegisteredRootsJob)); + scrrj->scan_job.ops = ops; + scrrj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue; + scrrj->heap_start = heap_start; + scrrj->heap_end = heap_end; + scrrj->root_type = ROOT_TYPE_WBARRIER; + sgen_workers_enqueue_job (&scrrj->scan_job.job, enqueue); + } /* Threads */ @@ -2599,6 +2612,94 @@ sgen_deregister_root (char* addr) UNLOCK_GC; } +void +sgen_wbroots_iterate_live_block_ranges (sgen_cardtable_block_callback cb) +{ + void **start_root; + RootRecord *root; + SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_WBARRIER], void **, start_root, RootRecord *, root) { + cb ((mword)start_root, (mword)root->end_root - (mword)start_root); + } SGEN_HASH_TABLE_FOREACH_END; +} + +/* Root equivalent of sgen_client_cardtable_scan_object */ +static void +sgen_wbroot_scan_card_table (void** start_root, mword size, ScanCopyContext ctx) +{ + ScanPtrFieldFunc scan_field_func = ctx.ops->scan_ptr_field; + guint8 *card_data = sgen_card_table_get_card_scan_address ((mword)start_root); + guint8 *card_base = card_data; + mword card_count = sgen_card_table_number_of_cards_in_range ((mword)start_root, size); + guint8 *card_data_end = card_data + card_count; + mword extra_idx = 0; + char *obj_start = sgen_card_table_align_pointer (start_root); + char *obj_end = (char*)start_root + size; +#ifdef SGEN_HAVE_OVERLAPPING_CARDS + guint8 *overflow_scan_end = NULL; +#endif + +#ifdef SGEN_HAVE_OVERLAPPING_CARDS + /*Check for overflow and if so, setup to scan in two steps*/ + if (card_data_end >= SGEN_SHADOW_CARDTABLE_END) { + overflow_scan_end = sgen_shadow_cardtable + (card_data_end - SGEN_SHADOW_CARDTABLE_END); + card_data_end = SGEN_SHADOW_CARDTABLE_END; + } + +LOOP_HEAD: +#endif + + card_data = sgen_find_next_card (card_data, card_data_end); + + for (; card_data < card_data_end; card_data = sgen_find_next_card (card_data + 1, card_data_end)) { + size_t idx = (card_data - card_base) + extra_idx; + char *start = (char*)(obj_start + idx * CARD_SIZE_IN_BYTES); + char *card_end = start + CARD_SIZE_IN_BYTES; + char *elem = start, *first_elem = start; + + /* + * Don't clean first and last card on 32bit systems since they + * may also be part from other roots. + */ + if (card_data != card_base && card_data != (card_data_end - 1)) + sgen_card_table_prepare_card_for_scanning (card_data); + + card_end = MIN (card_end, obj_end); + + if (elem < (char*)start_root) + first_elem = elem = (char*)start_root; + + for (; elem < card_end; elem += SIZEOF_VOID_P) { + if (*(GCObject**)elem) + scan_field_func (NULL, (GCObject**)elem, ctx.queue); + } + + binary_protocol_card_scan (first_elem, elem - first_elem); + } + +#ifdef SGEN_HAVE_OVERLAPPING_CARDS + if (overflow_scan_end) { + extra_idx = card_data - card_base; + card_base = card_data = sgen_shadow_cardtable; + card_data_end = overflow_scan_end; + overflow_scan_end = NULL; + goto LOOP_HEAD; + } +#endif +} + +void +sgen_wbroots_scan_card_table (ScanCopyContext ctx) +{ + void **start_root; + RootRecord *root; + + SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_WBARRIER], void **, start_root, RootRecord *, root) { + SGEN_ASSERT (0, (root->root_desc & ROOT_DESC_TYPE_MASK) == ROOT_DESC_VECTOR, "Unsupported root type"); + + sgen_wbroot_scan_card_table (start_root, (mword)root->end_root - (mword)start_root, ctx); + } SGEN_HASH_TABLE_FOREACH_END; +} + /* * ###################################################################### * ######## Thread handling (stop/start code) diff --git a/mono/sgen/sgen-gc.h b/mono/sgen/sgen-gc.h index ba7aee72940..a89a5471dd7 100644 --- a/mono/sgen/sgen-gc.h +++ b/mono/sgen/sgen-gc.h @@ -832,6 +832,9 @@ void sgen_finalize_if (SgenObjectPredicateFunc predicate, void *user_data); void sgen_remove_finalizers_if (SgenObjectPredicateFunc predicate, void *user_data, int generation); void sgen_set_suspend_finalizers (void); +void sgen_wbroots_iterate_live_block_ranges (sgen_cardtable_block_callback cb); +void sgen_wbroots_scan_card_table (ScanCopyContext ctx); + void sgen_register_disappearing_link (GCObject *obj, void **link, gboolean track, gboolean in_gc); GCObject* sgen_weak_link_get (void **link_addr); diff --git a/mono/sgen/sgen-marksweep-drain-gray-stack.h b/mono/sgen/sgen-marksweep-drain-gray-stack.h index 6eb4d266fdc..98d8917593c 100644 --- a/mono/sgen/sgen-marksweep-drain-gray-stack.h +++ b/mono/sgen/sgen-marksweep-drain-gray-stack.h @@ -240,7 +240,7 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ GCObject *__old = *(ptr); \ binary_protocol_scan_process_reference ((full_object), (ptr), __old); \ if (__old && !sgen_ptr_in_nursery (__old)) { \ - if (G_UNLIKELY (!sgen_ptr_in_nursery (ptr) && \ + if (G_UNLIKELY (full_object && !sgen_ptr_in_nursery (ptr) && \ sgen_safe_object_is_small (__old, sgen_obj_get_descriptor (__old) & DESC_TYPE_MASK) && \ major_block_is_evacuating (MS_BLOCK_FOR_OBJ (__old)))) { \ mark_mod_union_card ((full_object), (void**)(ptr), __old); \ @@ -249,7 +249,7 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \ } \ } else { \ - if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \ + if (G_UNLIKELY (full_object && sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \ mark_mod_union_card ((full_object), (void**)(ptr), __old); \ } \ } while (0) @@ -261,7 +261,7 @@ SCAN_OBJECT_FUNCTION_NAME (GCObject *full_object, SgenDescriptor desc, SgenGrayQ PREFETCH_READ (__old); \ COPY_OR_MARK_FUNCTION_NAME ((ptr), __old, queue); \ } else { \ - if (G_UNLIKELY (sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \ + if (G_UNLIKELY (full_object && sgen_ptr_in_nursery (__old) && !sgen_ptr_in_nursery ((ptr)) && !sgen_cement_is_forced (__old))) \ mark_mod_union_card ((full_object), (void**)(ptr), __old); \ } \ } while (0) @@ -310,6 +310,12 @@ SCAN_VTYPE_FUNCTION_NAME (GCObject *full_object, char *start, SgenDescriptor des static void SCAN_PTR_FIELD_FUNCTION_NAME (GCObject *full_object, GCObject **ptr, SgenGrayQueue *queue) { + /* + * full_object is NULL if we scan unmanaged memory. This means we can't mark + * mod unions for it, so these types of roots currently don't have support + * for the concurrent collector (aka they need to be scanned as normal roots + * both in the start and finishing pause) + */ HANDLE_PTR (ptr, NULL); } #endif diff --git a/mono/sgen/sgen-marksweep.c b/mono/sgen/sgen-marksweep.c index b280749a46a..5c4ec06a9eb 100644 --- a/mono/sgen/sgen-marksweep.c +++ b/mono/sgen/sgen-marksweep.c @@ -2809,6 +2809,7 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr collector->major_ops_serial.copy_or_mark_object = major_copy_or_mark_object_canonical; collector->major_ops_serial.scan_object = major_scan_object_with_evacuation; + collector->major_ops_serial.scan_ptr_field = major_scan_ptr_field_with_evacuation; collector->major_ops_serial.drain_gray_stack = drain_gray_stack; if (is_concurrent) { collector->major_ops_concurrent_start.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical; -- 2.25.1