X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Fmetadata%2Fsgen-new-bridge.c;h=51b00e96695475d26176387342f33ba37ae13695;hb=dd07fe7aaf26f0ecac3fe846218b6f960c7cbf9e;hp=d450e401581283d6dee0ed4c80b484b3de383a31;hpb=ee99d7e569653e707708d217f48eab8023e9cb96;p=mono.git diff --git a/mono/metadata/sgen-new-bridge.c b/mono/metadata/sgen-new-bridge.c index d450e401581..51b00e96695 100644 --- a/mono/metadata/sgen-new-bridge.c +++ b/mono/metadata/sgen-new-bridge.c @@ -42,19 +42,39 @@ #ifdef HAVE_SGEN_GC #include +#include #include "sgen-gc.h" #include "sgen-bridge.h" #include "sgen-hash-table.h" #include "sgen-qsort.h" +#include "tabledefs.h" #include "utils/mono-logger-internal.h" #include "utils/mono-time.h" #include "utils/mono-compiler.h" +//#define NEW_XREFS +#ifdef NEW_XREFS +//#define TEST_NEW_XREFS +#endif + +#if !defined(NEW_XREFS) || defined(TEST_NEW_XREFS) +#define OLD_XREFS +#endif + +#ifdef NEW_XREFS +#define XREFS new_xrefs +#else +#define XREFS old_xrefs +#endif + +#define OPTIMIZATION_COPY +#define OPTIMIZATION_FORWARD +#define OPTIMIZATION_SINGLETON_DYN_ARRAY typedef struct { int size; - int capacity; + int capacity; /* if negative, data points to another DynArray's data */ char *data; } DynArray; @@ -80,16 +100,20 @@ typedef struct { * just one source, so use the srcs pointer itself. */ typedef struct _HashEntry { - MonoObject *obj; /* This is a duplicate - it's already stored in the hash table */ - gboolean is_bridge; - gboolean is_visited; - int finishing_time; + union { + struct { + guint32 is_visited : 1; + guint32 finishing_time : 31; + struct _HashEntry *forwarded_to; + } dfs1; + struct { + int scc_index; + } dfs2; + } v; DynPtrArray srcs; - - int scc_index; } HashEntry; typedef struct { @@ -101,15 +125,27 @@ typedef struct _SCC { int index; int api_index; int num_bridge_entries; - DynIntArray xrefs; /* these are incoming, not outgoing */ + gboolean flag; + /* + * New and old xrefs are typically mutually exclusive. Only when TEST_NEW_XREFS is + * enabled we do both, and compare the results. This should only be done for + * debugging, obviously. + */ +#ifdef OLD_XREFS + DynIntArray old_xrefs; /* these are incoming, not outgoing */ +#endif +#ifdef NEW_XREFS + DynIntArray new_xrefs; +#endif } SCC; static SgenHashTable hash_table = SGEN_HASH_TABLE_INIT (INTERNAL_MEM_BRIDGE_HASH_TABLE, INTERNAL_MEM_BRIDGE_HASH_TABLE_ENTRY, sizeof (HashEntry), mono_aligned_addr_hash, NULL); -static int current_time; +static guint32 current_time; static gboolean bridge_accounting_enabled = FALSE; +static SgenBridgeProcessor *bridge_processor; /* Core functions */ /* public */ @@ -127,33 +163,65 @@ dyn_array_init (DynArray *da) static void dyn_array_uninit (DynArray *da, int elem_size) { - if (da->capacity <= 0) + if (da->capacity < 0) { + dyn_array_init (da); + return; + } + + if (da->capacity == 0) return; sgen_free_internal_dynamic (da->data, elem_size * da->capacity, INTERNAL_MEM_BRIDGE_DATA); da->data = NULL; } +static void +dyn_array_empty (DynArray *da) +{ + if (da->capacity < 0) + dyn_array_init (da); + else + da->size = 0; +} + static void dyn_array_ensure_capacity (DynArray *da, int capacity, int elem_size) { int old_capacity = da->capacity; char *new_data; + g_assert (capacity > 0); + if (capacity <= old_capacity) return; - if (da->capacity == 0) + if (old_capacity <= 0) da->capacity = 2; while (capacity > da->capacity) da->capacity *= 2; new_data = sgen_alloc_internal_dynamic (elem_size * da->capacity, INTERNAL_MEM_BRIDGE_DATA, TRUE); memcpy (new_data, da->data, elem_size * da->size); - sgen_free_internal_dynamic (da->data, elem_size * old_capacity, INTERNAL_MEM_BRIDGE_DATA); + if (old_capacity > 0) + sgen_free_internal_dynamic (da->data, elem_size * old_capacity, INTERNAL_MEM_BRIDGE_DATA); da->data = new_data; } +static gboolean +dyn_array_is_copy (DynArray *da) +{ + return da->capacity < 0; +} + +static void +dyn_array_ensure_independent (DynArray *da, int elem_size) +{ + if (!dyn_array_is_copy (da)) + return; + dyn_array_ensure_capacity (da, da->size, elem_size); + g_assert (da->capacity > 0); +} + static void* dyn_array_add (DynArray *da, int elem_size) { @@ -166,6 +234,19 @@ dyn_array_add (DynArray *da, int elem_size) return p; } +static void +dyn_array_copy (DynArray *dst, DynArray *src, int elem_size) +{ + dyn_array_uninit (dst, elem_size); + + if (src->size == 0) + return; + + dst->size = src->size; + dst->capacity = -1; + dst->data = src->data; +} + /* int */ static void dyn_array_int_init (DynIntArray *da) @@ -185,11 +266,13 @@ dyn_array_int_size (DynIntArray *da) return da->array.size; } +#ifdef NEW_XREFS static void -dyn_array_int_set_size (DynIntArray *da, int size) +dyn_array_int_empty (DynIntArray *da) { - da->array.size = size; + dyn_array_empty (&da->array); } +#endif static void dyn_array_int_add (DynIntArray *da, int x) @@ -204,24 +287,30 @@ dyn_array_int_get (DynIntArray *da, int x) return ((int*)da->array.data)[x]; } +#ifdef NEW_XREFS static void dyn_array_int_set (DynIntArray *da, int idx, int val) { ((int*)da->array.data)[idx] = val; } +#endif static void -dyn_array_int_ensure_capacity (DynIntArray *da, int capacity) +dyn_array_int_ensure_independent (DynIntArray *da) { - dyn_array_ensure_capacity (&da->array, capacity, sizeof (int)); + dyn_array_ensure_independent (&da->array, sizeof (int)); } static void -dyn_array_int_set_all (DynIntArray *dst, DynIntArray *src) +dyn_array_int_copy (DynIntArray *dst, DynIntArray *src) +{ + dyn_array_copy (&dst->array, &src->array, sizeof (int)); +} + +static gboolean +dyn_array_int_is_copy (DynIntArray *da) { - dyn_array_int_ensure_capacity (dst, src->array.size); - memcpy (dst->array.data, src->array.data, src->array.size * sizeof (int)); - dst->array.size = src->array.size; + return dyn_array_is_copy (&da->array); } /* ptr */ @@ -235,7 +324,12 @@ dyn_array_ptr_init (DynPtrArray *da) static void dyn_array_ptr_uninit (DynPtrArray *da) { - dyn_array_uninit (&da->array, sizeof (void*)); +#ifdef OPTIMIZATION_SINGLETON_DYN_ARRAY + if (da->array.capacity == 1) + dyn_array_ptr_init (da); + else +#endif + dyn_array_uninit (&da->array, sizeof (void*)); } static int @@ -245,21 +339,50 @@ dyn_array_ptr_size (DynPtrArray *da) } static void -dyn_array_ptr_set_size (DynPtrArray *da, int size) +dyn_array_ptr_empty (DynPtrArray *da) { - da->array.size = size; +#ifdef OPTIMIZATION_SINGLETON_DYN_ARRAY + if (da->array.capacity == 1) + dyn_array_ptr_init (da); + else +#endif + dyn_array_empty (&da->array); } static void* dyn_array_ptr_get (DynPtrArray *da, int x) { +#ifdef OPTIMIZATION_SINGLETON_DYN_ARRAY + if (da->array.capacity == 1) { + g_assert (x == 0); + return da->array.data; + } +#endif return ((void**)da->array.data)[x]; } static void dyn_array_ptr_add (DynPtrArray *da, void *ptr) { - void **p = dyn_array_add (&da->array, sizeof (void*)); + void **p; + +#ifdef OPTIMIZATION_SINGLETON_DYN_ARRAY + if (da->array.capacity == 0) { + da->array.capacity = 1; + da->array.size = 1; + p = (void**)&da->array.data; + } else if (da->array.capacity == 1) { + void *ptr0 = da->array.data; + void **p0; + dyn_array_init (&da->array); + p0 = dyn_array_add (&da->array, sizeof (void*)); + *p0 = ptr0; + p = dyn_array_add (&da->array, sizeof (void*)); + } else +#endif + { + p = dyn_array_add (&da->array, sizeof (void*)); + } *p = ptr; } @@ -268,11 +391,21 @@ dyn_array_ptr_add (DynPtrArray *da, void *ptr) static void* dyn_array_ptr_pop (DynPtrArray *da) { - void *p; int size = da->array.size; + void *p; g_assert (size > 0); - p = dyn_array_ptr_get (da, size - 1); - --da->array.size; +#ifdef OPTIMIZATION_SINGLETON_DYN_ARRAY + if (da->array.capacity == 1) { + p = dyn_array_ptr_get (da, 0); + dyn_array_init (&da->array); + } else +#endif + { + g_assert (da->array.capacity > 1); + dyn_array_ensure_independent (&da->array, sizeof (void*)); + p = dyn_array_ptr_get (da, size - 1); + --da->array.size; + } return p; } @@ -312,6 +445,7 @@ dyn_array_scc_get_ptr (DynSCCArray *da, int x) static DynIntArray merge_array; +#ifdef NEW_XREFS static gboolean dyn_array_int_contains (DynIntArray *da, int x) { @@ -321,88 +455,44 @@ dyn_array_int_contains (DynIntArray *da, int x) return TRUE; return FALSE; } - +#endif static void -dyn_array_int_merge (DynIntArray *dst, DynIntArray *src) +enable_accounting (void) { - int i, j; - - dyn_array_int_ensure_capacity (&merge_array, dyn_array_int_size (dst) + dyn_array_int_size (src)); - dyn_array_int_set_size (&merge_array, 0); - - for (i = j = 0; i < dyn_array_int_size (dst) || j < dyn_array_int_size (src); ) { - if (i < dyn_array_int_size (dst) && j < dyn_array_int_size (src)) { - int a = dyn_array_int_get (dst, i); - int b = dyn_array_int_get (src, j); - if (a < b) { - dyn_array_int_add (&merge_array, a); - ++i; - } else if (a == b) { - dyn_array_int_add (&merge_array, a); - ++i; - ++j; - } else { - dyn_array_int_add (&merge_array, b); - ++j; - } - } else if (i < dyn_array_int_size (dst)) { - dyn_array_int_add (&merge_array, dyn_array_int_get (dst, i)); - ++i; - } else { - dyn_array_int_add (&merge_array, dyn_array_int_get (src, j)); - ++j; - } - } - - if (dyn_array_int_size (&merge_array) > dyn_array_int_size (dst)) { - dyn_array_int_set_all (dst, &merge_array); - } + SgenHashTable table = SGEN_HASH_TABLE_INIT (INTERNAL_MEM_BRIDGE_HASH_TABLE, INTERNAL_MEM_BRIDGE_HASH_TABLE_ENTRY, sizeof (HashEntryWithAccounting), mono_aligned_addr_hash, NULL); + bridge_accounting_enabled = TRUE; + hash_table = table; } -static void -dyn_array_int_merge_one (DynIntArray *array, int value) +static MonoGCBridgeObjectKind +class_kind (MonoClass *class) { - int i; - int tmp; - int size = dyn_array_int_size (array); + MonoGCBridgeObjectKind res = bridge_callbacks.bridge_class_kind (class); - for (i = 0; i < size; ++i) { - if (dyn_array_int_get (array, i) == value) - return; - else if (dyn_array_int_get (array, i) > value) - break; + /* If it's a bridge, nothing we can do about it. */ + if (res == GC_BRIDGE_TRANSPARENT_BRIDGE_CLASS || res == GC_BRIDGE_OPAQUE_BRIDGE_CLASS) + return res; + + /* Non bridge classes with no pointers will never point to a bridge, so we can savely ignore them. */ + if (!class->has_references) { + SGEN_LOG (6, "class %s is opaque\n", class->name); + return GC_BRIDGE_OPAQUE_CLASS; } - dyn_array_int_ensure_capacity (array, size + 1); + /* Some arrays can be ignored */ + if (class->rank == 1) { + MonoClass *elem_class = class->element_class; - if (i < size) { - tmp = dyn_array_int_get (array, i); - for (; i < size; ++i) { - dyn_array_int_set (array, i, value); - value = tmp; - tmp = dyn_array_int_get (array, i + 1); + /* FIXME the bridge check can be quite expensive, cache it at the class level. */ + /* An array of a sealed type that is not a bridge will never get to a bridge */ + if ((elem_class->flags & TYPE_ATTRIBUTE_SEALED) && !elem_class->has_references && !bridge_callbacks.bridge_class_kind (elem_class)) { + SGEN_LOG (6, "class %s is opaque\n", class->name); + return GC_BRIDGE_OPAQUE_CLASS; } - dyn_array_int_set (array, size, value); - } else { - dyn_array_int_set (array, size, value); } - dyn_array_int_set_size (array, size + 1); -} - - -static void -enable_accounting (void) -{ - bridge_accounting_enabled = TRUE; - hash_table = (SgenHashTable)SGEN_HASH_TABLE_INIT (INTERNAL_MEM_BRIDGE_HASH_TABLE, INTERNAL_MEM_BRIDGE_HASH_TABLE_ENTRY, sizeof (HashEntryWithAccounting), mono_aligned_addr_hash, NULL); -} - -static MonoGCBridgeObjectKind -class_kind (MonoClass *class) -{ - return bridge_callbacks.bridge_class_kind (class); + return GC_BRIDGE_TRANSPARENT_CLASS; } static HashEntry* @@ -421,10 +511,8 @@ get_hash_entry (MonoObject *obj, gboolean *existing) memset (&new_entry, 0, sizeof (HashEntry)); - new_entry.obj = obj; dyn_array_ptr_init (&new_entry.srcs); - new_entry.finishing_time = -1; - new_entry.scc_index = -1; + new_entry.v.dfs1.finishing_time = 0; sgen_hash_table_replace (&hash_table, obj, &new_entry, NULL); @@ -468,24 +556,59 @@ register_bridge_object (MonoObject *obj) } static void -register_finishing_time (HashEntry *entry, int t) +register_finishing_time (HashEntry *entry, guint32 t) { - g_assert (entry->finishing_time < 0); - entry->finishing_time = t; + g_assert (entry->v.dfs1.finishing_time == 0); + /* finishing_time has 31 bits, so it must be within signed int32 range. */ + g_assert (t > 0 && t <= G_MAXINT32); + entry->v.dfs1.finishing_time = t; } +static int ignored_objects; + static gboolean -object_is_live (MonoObject **objp) +is_opaque_object (MonoObject *obj) +{ + if ((obj->vtable->gc_bits & SGEN_GC_BIT_BRIDGE_OPAQUE_OBJECT) == SGEN_GC_BIT_BRIDGE_OPAQUE_OBJECT) { + SGEN_LOG (6, "ignoring %s\n", obj->vtable->klass->name); + ++ignored_objects; + return TRUE; + } + return FALSE; +} + +static gboolean +object_needs_expansion (MonoObject **objp) { MonoObject *obj = *objp; MonoObject *fwd = SGEN_OBJECT_IS_FORWARDED (obj); if (fwd) { *objp = fwd; - return sgen_hash_table_lookup (&hash_table, fwd) == NULL; + if (is_opaque_object (fwd)) + return FALSE; + return sgen_hash_table_lookup (&hash_table, fwd) != NULL; } - if (!sgen_object_is_live (obj)) + if (is_opaque_object (obj)) return FALSE; - return sgen_hash_table_lookup (&hash_table, obj) == NULL; + if (!sgen_object_is_live (obj)) + return TRUE; + return sgen_hash_table_lookup (&hash_table, obj) != NULL; +} + +static HashEntry* +follow_forward (HashEntry *entry) +{ +#ifdef OPTIMIZATION_FORWARD + while (entry->v.dfs1.forwarded_to) { + HashEntry *next = entry->v.dfs1.forwarded_to; + if (next->v.dfs1.forwarded_to) + entry->v.dfs1.forwarded_to = next->v.dfs1.forwarded_to; + entry = next; + } +#else + g_assert (!entry->v.dfs1.forwarded_to); +#endif + return entry; } static DynPtrArray registered_bridges; @@ -493,13 +616,21 @@ static DynPtrArray dfs_stack; static int dfs1_passes, dfs2_passes; +/* + * DFS1 maintains a stack, where each two entries are effectively one entry. (FIXME: + * Optimize this via pointer tagging.) There are two different types of entries: + * + * entry, src: entry needs to be expanded via scanning, and linked to from src + * NULL, entry: entry has already been expanded and needs to be finished + */ #undef HANDLE_PTR #define HANDLE_PTR(ptr,obj) do { \ MonoObject *dst = (MonoObject*)*(ptr); \ - if (dst && !object_is_live (&dst)) { \ + if (dst && object_needs_expansion (&dst)) { \ + ++num_links; \ dyn_array_ptr_push (&dfs_stack, obj_entry); \ - dyn_array_ptr_push (&dfs_stack, get_hash_entry (dst, NULL)); \ + dyn_array_ptr_push (&dfs_stack, follow_forward (get_hash_entry (dst, NULL))); \ } \ } while (0) @@ -519,67 +650,185 @@ dfs1 (HashEntry *obj_entry) obj_entry = dyn_array_ptr_pop (&dfs_stack); if (obj_entry) { + /* obj_entry needs to be expanded */ src = dyn_array_ptr_pop (&dfs_stack); - obj = obj_entry->obj; + if (src) + g_assert (!src->v.dfs1.forwarded_to); + + obj_entry = follow_forward (obj_entry); + + again: + g_assert (!obj_entry->v.dfs1.forwarded_to); + obj = sgen_hash_table_key_for_value_pointer (obj_entry); start = (char*)obj; + if (!obj_entry->v.dfs1.is_visited) { + int num_links = 0; + mword desc = sgen_obj_get_descriptor (start); + + obj_entry->v.dfs1.is_visited = 1; + + /* push the finishing entry on the stack */ + dyn_array_ptr_push (&dfs_stack, obj_entry); + dyn_array_ptr_push (&dfs_stack, NULL); + +#include "sgen-scan-object.h" + + /* + * We can remove non-bridge objects with a single outgoing + * link by forwarding links going to it. + * + * This is the first time we've encountered this object, so + * no links to it have yet been added. We'll keep it that + * way by setting the forward pointer, and instead of + * continuing processing this object, we start over with the + * object it points to. + */ +#ifdef OPTIMIZATION_FORWARD + if (!obj_entry->is_bridge && num_links == 1) { + HashEntry *dst_entry = dyn_array_ptr_pop (&dfs_stack); + HashEntry *obj_entry_again = dyn_array_ptr_pop (&dfs_stack); + g_assert (obj_entry_again == obj_entry); + g_assert (!dst_entry->v.dfs1.forwarded_to); + if (obj_entry != dst_entry) { + obj_entry->v.dfs1.forwarded_to = dst_entry; + obj_entry = dst_entry; + } + goto again; + } +#endif + } + if (src) { //g_print ("link %s -> %s\n", sgen_safe_name (src->obj), sgen_safe_name (obj)); + g_assert (!obj_entry->v.dfs1.forwarded_to); add_source (obj_entry, src); } else { //g_print ("starting with %s\n", sgen_safe_name (obj)); } - - if (obj_entry->is_visited) - continue; - - obj_entry->is_visited = TRUE; - - dyn_array_ptr_push (&dfs_stack, obj_entry); - /* NULL marks that the next entry is to be finished */ - dyn_array_ptr_push (&dfs_stack, NULL); - -#include "sgen-scan-object.h" } else { + /* obj_entry needs to be finished */ + obj_entry = dyn_array_ptr_pop (&dfs_stack); //g_print ("finish %s\n", sgen_safe_name (obj_entry->obj)); - register_finishing_time (obj_entry, current_time++); + register_finishing_time (obj_entry, ++current_time); } } while (dyn_array_ptr_size (&dfs_stack) > 0); } +static DynSCCArray sccs; +static SCC *current_scc; + +/* + * At the end of bridge processing we need to end up with an (acyclyc) graph of bridge + * object SCCs, where the links between the nodes (each one an SCC) in that graph represent + * the presence of a direct or indirect link between those SCCs. An example: + * + * D + * | + * v + * A -> B -> c -> e -> F + * + * A, B, D and F are SCCs that contain bridge objects, c and e don't contain bridge objects. + * The graph we need to produce from this is: + * + * D + * | + * v + * A -> B -> F + * + * Note that we don't need to produce an edge from A to F. It's sufficient that F is + * indirectly reachable from A. + * + * The old algorithm would create a set, for each SCC, of bridge SCCs that can reach it, + * directly or indirectly, by merging the ones sets for those that reach it directly. The + * sets it would build up are these: + * + * A: {} + * B: {A} + * c: {B} + * D: {} + * e: {B,D} + * F: {B,D} + * + * The merge operations on these sets turned out to be huge time sinks. + * + * The new algorithm proceeds in two passes: During DFS2, it only builds up the sets of SCCs + * that directly point to each SCC: + * + * A: {} + * B: {A} + * c: {B} + * D: {} + * e: {c,D} + * F: {e} + * + * This is the adjacency list for the SCC graph, in other words. In a separate step + * afterwards, it does a depth-first traversal of that graph, for each bridge node, to get + * to the final list. It uses a flag to avoid traversing any node twice. + */ static void scc_add_xref (SCC *src, SCC *dst) { g_assert (src != dst); g_assert (src->index != dst->index); - if (dyn_array_int_contains (&dst->xrefs, src->index)) - return; +#ifdef NEW_XREFS + /* + * FIXME: Right now we don't even unique the direct ancestors, but just add to the + * list. Doing a containment check slows this algorithm down to almost the speed of + * the old one. Use the flag instead! + */ + dyn_array_int_add (&dst->new_xrefs, src->index); +#endif + +#ifdef OLD_XREFS + if (dyn_array_int_is_copy (&dst->old_xrefs)) { + int i; + dyn_array_int_ensure_independent (&dst->old_xrefs); + for (i = 0; i < dyn_array_int_size (&dst->old_xrefs); ++i) { + int j = dyn_array_int_get (&dst->old_xrefs, i); + SCC *bridge_scc = dyn_array_scc_get_ptr (&sccs, j); + g_assert (!bridge_scc->flag); + bridge_scc->flag = TRUE; + } + } + if (src->num_bridge_entries) { - dyn_array_int_merge_one (&dst->xrefs, src->index); + if (src->flag) + return; + src->flag = TRUE; + dyn_array_int_add (&dst->old_xrefs, src->index); +#ifdef OPTIMIZATION_COPY + } else if (dyn_array_int_size (&dst->old_xrefs) == 0) { + dyn_array_int_copy (&dst->old_xrefs, &src->old_xrefs); +#endif } else { int i; - dyn_array_int_merge (&dst->xrefs, &src->xrefs); - for (i = 0; i < dyn_array_int_size (&dst->xrefs); ++i) - g_assert (dyn_array_int_get (&dst->xrefs, i) != dst->index); + for (i = 0; i < dyn_array_int_size (&src->old_xrefs); ++i) { + int j = dyn_array_int_get (&src->old_xrefs, i); + SCC *bridge_scc = dyn_array_scc_get_ptr (&sccs, j); + g_assert (bridge_scc->num_bridge_entries); + if (!bridge_scc->flag) { + bridge_scc->flag = TRUE; + dyn_array_int_add (&dst->old_xrefs, j); + } + } } +#endif } static void scc_add_entry (SCC *scc, HashEntry *entry) { - g_assert (entry->scc_index < 0); - entry->scc_index = scc->index; + g_assert (entry->v.dfs2.scc_index < 0); + entry->v.dfs2.scc_index = scc->index; if (entry->is_bridge) ++scc->num_bridge_entries; } -static DynSCCArray sccs; -static SCC *current_scc; - static void dfs2 (HashEntry *entry) { @@ -593,9 +842,9 @@ dfs2 (HashEntry *entry) entry = dyn_array_ptr_pop (&dfs_stack); ++dfs2_passes; - if (entry->scc_index >= 0) { - if (entry->scc_index != current_scc->index) - scc_add_xref (dyn_array_scc_get_ptr (&sccs, entry->scc_index), current_scc); + if (entry->v.dfs2.scc_index >= 0) { + if (entry->v.dfs2.scc_index != current_scc->index) + scc_add_xref (dyn_array_scc_get_ptr (&sccs, entry->v.dfs2.scc_index), current_scc); continue; } @@ -604,17 +853,123 @@ dfs2 (HashEntry *entry) for (i = 0; i < dyn_array_ptr_size (&entry->srcs); ++i) dyn_array_ptr_push (&dfs_stack, dyn_array_ptr_get (&entry->srcs, i)); } while (dyn_array_ptr_size (&dfs_stack) > 0); + +#ifdef OLD_XREFS + /* If xrefs is a copy then we haven't set a single flag. */ + if (dyn_array_int_is_copy (¤t_scc->old_xrefs)) + return; + for (i = 0; i < dyn_array_int_size (¤t_scc->old_xrefs); ++i) { + int j = dyn_array_int_get (¤t_scc->old_xrefs, i); + SCC *bridge_scc = dyn_array_scc_get_ptr (&sccs, j); + g_assert (bridge_scc->flag); + bridge_scc->flag = FALSE; + } +#endif +} + +#ifdef NEW_XREFS +static void +gather_xrefs (SCC *scc) +{ + int i; + for (i = 0; i < dyn_array_int_size (&scc->new_xrefs); ++i) { + int index = dyn_array_int_get (&scc->new_xrefs, i); + SCC *src = dyn_array_scc_get_ptr (&sccs, index); + if (src->flag) + continue; + src->flag = TRUE; + if (src->num_bridge_entries) + dyn_array_int_add (&merge_array, index); + else + gather_xrefs (src); + } +} + +static void +reset_flags (SCC *scc) +{ + int i; + for (i = 0; i < dyn_array_int_size (&scc->new_xrefs); ++i) { + int index = dyn_array_int_get (&scc->new_xrefs, i); + SCC *src = dyn_array_scc_get_ptr (&sccs, index); + if (!src->flag) + continue; + src->flag = FALSE; + if (!src->num_bridge_entries) + reset_flags (src); + } +} +#endif + +static char *dump_prefix = NULL; + +static void +dump_graph (void) +{ + static int counter = 0; + + MonoObject *obj; + HashEntry *entry; + size_t prefix_len = strlen (dump_prefix); + char *filename = alloca(prefix_len + 64); + FILE *file; + int edge_id = 0; + + sprintf (filename, "%s.%d.gexf", dump_prefix, counter++); + file = fopen (filename, "w"); + + if (file == NULL) { + fprintf (stderr, "Warning: Could not open bridge dump file `%s` for writing: %s\n", filename, strerror (errno)); + return; + } + + fprintf (file, "\n"); + + fprintf (file, "\n" + "\n" + "\n" + "\n" + "\n"); + + fprintf (file, "\n"); + SGEN_HASH_TABLE_FOREACH (&hash_table, obj, entry) { + MonoVTable *vt = (MonoVTable*) SGEN_LOAD_VTABLE (obj); + fprintf (file, "\n", + obj, vt->klass->name_space, vt->klass->name, entry->is_bridge ? "true" : "false"); + } SGEN_HASH_TABLE_FOREACH_END; + fprintf (file, "\n"); + + fprintf (file, "\n"); + SGEN_HASH_TABLE_FOREACH (&hash_table, obj, entry) { + int i; + for (i = 0; i < dyn_array_ptr_size (&entry->srcs); ++i) { + HashEntry *src = dyn_array_ptr_get (&entry->srcs, i); + fprintf (file, "\n", edge_id++, sgen_hash_table_key_for_value_pointer (src), obj); + } + } SGEN_HASH_TABLE_FOREACH_END; + fprintf (file, "\n"); + + fprintf (file, "\n"); + + fclose (file); +} + +static void +set_dump_prefix (const char *prefix) +{ + dump_prefix = strdup (prefix); } static int compare_hash_entries (const HashEntry *e1, const HashEntry *e2) { - return e2->finishing_time - e1->finishing_time; + /* We can cast to signed int here because finishing_time has only 31 bits. */ + return (gint32)e2->v.dfs1.finishing_time - (gint32)e1->v.dfs1.finishing_time; } DEF_QSORT_INLINE(hash_entries, HashEntry*, compare_hash_entries) -static unsigned long step_1, step_2, step_3, step_4, step_5, step_6, step_7, step_8; +static unsigned long step_1, step_2, step_3, step_4, step_5, step_6; static int fist_pass_links, second_pass_links, sccs_links; static int max_sccs_links = 0; @@ -628,7 +983,7 @@ register_finalized_object (MonoObject *obj) static void reset_data (void) { - dyn_array_ptr_set_size (®istered_bridges, 0); + dyn_array_ptr_empty (®istered_bridges); } static void @@ -636,18 +991,14 @@ processing_stw_step (void) { int i; int bridge_count; + MonoObject *obj; + HashEntry *entry; SGEN_TV_DECLARE (atv); SGEN_TV_DECLARE (btv); if (!dyn_array_ptr_size (®istered_bridges)) return; - /* - * bridge_processing_in_progress must be set with the world - * stopped. If not there would be race conditions. - */ - bridge_processing_in_progress = TRUE; - SGEN_TV_GETTIME (btv); /* first DFS pass */ @@ -672,37 +1023,42 @@ processing_stw_step (void) for (i = 0; i < bridge_count; ++i) dfs1 (get_hash_entry (dyn_array_ptr_get (®istered_bridges, i), NULL)); + /* Remove all forwarded objects. */ + SGEN_HASH_TABLE_FOREACH (&hash_table, obj, entry) { + if (entry->v.dfs1.forwarded_to) { + g_assert (dyn_array_ptr_size (&entry->srcs) == 0); + SGEN_HASH_TABLE_FOREACH_REMOVE (TRUE); + continue; + } + } SGEN_HASH_TABLE_FOREACH_END; + SGEN_TV_GETTIME (atv); step_2 = SGEN_TV_ELAPSED (btv, atv); -} -static mono_bool -is_bridge_object_alive (MonoObject *obj, void *data) -{ - SgenHashTable *table = data; - unsigned char *value = sgen_hash_table_lookup (table, obj); - if (!value) - return TRUE; - return *value; + if (dump_prefix) + dump_graph (); } +static int num_registered_bridges, hash_table_size; + static void -processing_finish (int generation) +processing_build_callback_data (int generation) { int i, j; int num_sccs, num_xrefs; int max_entries, max_xrefs; - int hash_table_size, sccs_size; + int sccs_size; MonoObject *obj; HashEntry *entry; - int num_registered_bridges; HashEntry **all_entries; MonoGCBridgeSCC **api_sccs; MonoGCBridgeXRef *api_xrefs; - SgenHashTable alive_hash = SGEN_HASH_TABLE_INIT (INTERNAL_MEM_BRIDGE_ALIVE_HASH_TABLE, INTERNAL_MEM_BRIDGE_ALIVE_HASH_TABLE_ENTRY, 1, mono_aligned_addr_hash, NULL); SGEN_TV_DECLARE (atv); SGEN_TV_DECLARE (btv); + g_assert (bridge_processor->num_sccs == 0 && bridge_processor->num_xrefs == 0); + g_assert (!bridge_processor->api_sccs && !bridge_processor->api_xrefs); + if (!dyn_array_ptr_size (®istered_bridges)) return; @@ -716,7 +1072,7 @@ processing_finish (int generation) j = 0; SGEN_HASH_TABLE_FOREACH (&hash_table, obj, entry) { - g_assert (entry->finishing_time >= 0); + g_assert (entry->v.dfs1.finishing_time > 0); all_entries [j++] = entry; fist_pass_links += dyn_array_ptr_size (&entry->srcs); } SGEN_HASH_TABLE_FOREACH_END; @@ -726,6 +1082,10 @@ processing_finish (int generation) /* sort array according to decreasing finishing time */ qsort_hash_entries (all_entries, hash_table.num_entries); + SGEN_HASH_TABLE_FOREACH (&hash_table, obj, entry) { + entry->v.dfs2.scc_index = -1; + } SGEN_HASH_TABLE_FOREACH_END; + SGEN_TV_GETTIME (btv); step_3 = SGEN_TV_ELAPSED (atv, btv); @@ -734,18 +1094,83 @@ processing_finish (int generation) dyn_array_scc_init (&sccs); for (i = 0; i < hash_table.num_entries; ++i) { HashEntry *entry = all_entries [i]; - if (entry->scc_index < 0) { + if (entry->v.dfs2.scc_index < 0) { int index = dyn_array_scc_size (&sccs); current_scc = dyn_array_scc_add (&sccs); current_scc->index = index; current_scc->num_bridge_entries = 0; +#ifdef NEW_XREFS + current_scc->flag = FALSE; + dyn_array_int_init (¤t_scc->new_xrefs); +#endif +#ifdef OLD_XREFS + dyn_array_int_init (¤t_scc->old_xrefs); +#endif current_scc->api_index = -1; - dyn_array_int_init (¤t_scc->xrefs); dfs2 (entry); + +#ifdef NEW_XREFS + /* + * If a node has only one incoming edge, we just copy the source's + * xrefs array, effectively removing the source from the graph. + * This takes care of long linked lists. + */ + if (!current_scc->num_bridge_entries && dyn_array_int_size (¤t_scc->new_xrefs) == 1) { + SCC *src; + j = dyn_array_int_get (¤t_scc->new_xrefs, 0); + src = dyn_array_scc_get_ptr (&sccs, j); + if (src->num_bridge_entries) + dyn_array_int_set (¤t_scc->new_xrefs, 0, j); + else + dyn_array_int_copy (¤t_scc->new_xrefs, &src->new_xrefs); + } +#endif + } + } + +#ifdef NEW_XREFS +#ifdef TEST_NEW_XREFS + for (j = 0; j < dyn_array_scc_size (&sccs); ++j) { + SCC *scc = dyn_array_scc_get_ptr (&sccs, j); + g_assert (!scc->flag); + } +#endif + + for (i = 0; i < dyn_array_scc_size (&sccs); ++i) { + SCC *scc = dyn_array_scc_get_ptr (&sccs, i); + g_assert (scc->index == i); + if (!scc->num_bridge_entries) + continue; + + dyn_array_int_empty (&merge_array); + gather_xrefs (scc); + reset_flags (scc); + dyn_array_int_copy (&scc->new_xrefs, &merge_array); + dyn_array_int_ensure_independent (&scc->new_xrefs); + +#ifdef TEST_NEW_XREFS + for (j = 0; j < dyn_array_scc_size (&sccs); ++j) { + SCC *scc = dyn_array_scc_get_ptr (&sccs, j); + g_assert (!scc->flag); } +#endif } +#ifdef TEST_NEW_XREFS + for (i = 0; i < dyn_array_scc_size (&sccs); ++i) { + SCC *scc = dyn_array_scc_get_ptr (&sccs, i); + g_assert (scc->index == i); + if (!scc->num_bridge_entries) + continue; + + g_assert (dyn_array_int_size (&scc->new_xrefs) == dyn_array_int_size (&scc->old_xrefs)); + for (j = 0; j < dyn_array_int_size (&scc->new_xrefs); ++j) + g_assert (dyn_array_int_contains (&scc->old_xrefs, dyn_array_int_get (&scc->new_xrefs, j))); + } +#endif +#endif + /* * Compute the weight of each object. The weight of an object is its size plus the size of all * objects it points do. When the an object is pointed by multiple objects we distribute it's weight @@ -767,7 +1192,7 @@ processing_finish (int generation) double w; HashEntryWithAccounting *entry = (HashEntryWithAccounting*)all_entries [i]; - entry->weight += (double)sgen_safe_object_get_size (entry->entry.obj); + entry->weight += (double)sgen_safe_object_get_size (sgen_hash_table_key_for_value_pointer (entry)); w = entry->weight / dyn_array_ptr_size (&entry->entry.srcs); for (j = 0; j < dyn_array_ptr_size (&entry->entry.srcs); ++j) { HashEntryWithAccounting *other = (HashEntryWithAccounting *)dyn_array_ptr_get (&entry->entry.srcs, j); @@ -777,8 +1202,9 @@ processing_finish (int generation) for (i = 0; i < hash_table.num_entries; ++i) { HashEntryWithAccounting *entry = (HashEntryWithAccounting*)all_entries [i]; if (entry->entry.is_bridge) { - MonoClass *klass = ((MonoVTable*)SGEN_LOAD_VTABLE (entry->entry.obj))->klass; - mono_trace (G_LOG_LEVEL_INFO, MONO_TRACE_GC, "OBJECT %s::%s (%p) weight %f", klass->name_space, klass->name, entry->entry.obj, entry->weight); + MonoObject *obj = sgen_hash_table_key_for_value_pointer (entry); + MonoClass *klass = ((MonoVTable*)SGEN_LOAD_VTABLE (obj))->klass; + mono_trace (G_LOG_LEVEL_INFO, MONO_TRACE_GC, "OBJECT %s::%s (%p) weight %f", klass->name_space, klass->name, obj, entry->weight); } } } @@ -805,8 +1231,8 @@ processing_finish (int generation) g_assert (scc->index == i); if (scc->num_bridge_entries) ++num_sccs; - sccs_links += dyn_array_int_size (&scc->xrefs); - max_sccs_links = MAX (max_sccs_links, dyn_array_int_size (&scc->xrefs)); + sccs_links += dyn_array_int_size (&scc->XREFS); + max_sccs_links = MAX (max_sccs_links, dyn_array_int_size (&scc->XREFS)); } api_sccs = sgen_alloc_internal_dynamic (sizeof (MonoGCBridgeSCC*) * num_sccs, INTERNAL_MEM_BRIDGE_DATA, TRUE); @@ -823,13 +1249,13 @@ processing_finish (int generation) scc->num_bridge_entries = 0; scc->api_index = j++; - num_xrefs += dyn_array_int_size (&scc->xrefs); + num_xrefs += dyn_array_int_size (&scc->XREFS); } SGEN_HASH_TABLE_FOREACH (&hash_table, obj, entry) { if (entry->is_bridge) { - SCC *scc = dyn_array_scc_get_ptr (&sccs, entry->scc_index); - api_sccs [scc->api_index]->objs [scc->num_bridge_entries++] = entry->obj; + SCC *scc = dyn_array_scc_get_ptr (&sccs, entry->v.dfs2.scc_index); + api_sccs [scc->api_index]->objs [scc->num_bridge_entries++] = sgen_hash_table_key_for_value_pointer (entry); } } SGEN_HASH_TABLE_FOREACH_END; @@ -840,8 +1266,8 @@ processing_finish (int generation) SCC *scc = dyn_array_scc_get_ptr (&sccs, i); if (!scc->num_bridge_entries) continue; - for (k = 0; k < dyn_array_int_size (&scc->xrefs); ++k) { - SCC *src_scc = dyn_array_scc_get_ptr (&sccs, dyn_array_int_get (&scc->xrefs, k)); + for (k = 0; k < dyn_array_int_size (&scc->XREFS); ++k) { + SCC *src_scc = dyn_array_scc_get_ptr (&sccs, dyn_array_int_get (&scc->XREFS, k)); if (!src_scc->num_bridge_entries) continue; api_xrefs [j].src_scc_index = src_scc->api_index; @@ -863,9 +1289,14 @@ processing_finish (int generation) ++j; if (scc->num_bridge_entries > max_entries) max_entries = scc->num_bridge_entries; - if (dyn_array_int_size (&scc->xrefs) > max_xrefs) - max_xrefs = dyn_array_int_size (&scc->xrefs); - dyn_array_int_uninit (&scc->xrefs); + if (dyn_array_int_size (&scc->XREFS) > max_xrefs) + max_xrefs = dyn_array_int_size (&scc->XREFS); +#ifdef NEW_XREFS + dyn_array_int_uninit (&scc->new_xrefs); +#endif +#ifdef OLD_XREFS + dyn_array_int_uninit (&scc->old_xrefs); +#endif } dyn_array_scc_uninit (&sccs); @@ -875,39 +1306,25 @@ processing_finish (int generation) free_data (); /* Empty the registered bridges array */ num_registered_bridges = dyn_array_ptr_size (®istered_bridges); - dyn_array_ptr_set_size (®istered_bridges, 0); + dyn_array_ptr_empty (®istered_bridges); SGEN_TV_GETTIME (atv); step_6 = SGEN_TV_ELAPSED (btv, atv); //g_print ("%d sccs containing bridges - %d max bridge objects - %d max xrefs\n", j, max_entries, max_xrefs); - /* callback */ - - bridge_callbacks.cross_references (num_sccs, api_sccs, num_xrefs, api_xrefs); - - /* Release for finalization those objects we no longer care. */ - SGEN_TV_GETTIME (btv); - step_7 = SGEN_TV_ELAPSED (atv, btv); - - for (i = 0; i < num_sccs; ++i) { - unsigned char alive = api_sccs [i]->is_alive ? 1 : 0; - for (j = 0; j < api_sccs [i]->num_objs; ++j) { - /* Build hash table for nulling weak links. */ - sgen_hash_table_replace (&alive_hash, api_sccs [i]->objs [j], &alive, NULL); - - /* Release for finalization those objects we no longer care. */ - if (!api_sccs [i]->is_alive) - sgen_mark_bridge_object (api_sccs [i]->objs [j]); - } - } - - /* Null weak links to dead objects. */ - sgen_null_links_with_predicate (GENERATION_NURSERY, is_bridge_object_alive, &alive_hash); - if (generation == GENERATION_OLD) - sgen_null_links_with_predicate (GENERATION_OLD, is_bridge_object_alive, &alive_hash); + bridge_processor->num_sccs = num_sccs; + bridge_processor->api_sccs = api_sccs; + bridge_processor->num_xrefs = num_xrefs; + bridge_processor->api_xrefs = api_xrefs; +} - sgen_hash_table_clean (&alive_hash); +static void +processing_after_callback (int generation) +{ + int i, j; + int num_sccs = bridge_processor->num_sccs; + MonoGCBridgeSCC **api_sccs = bridge_processor->api_sccs; if (bridge_accounting_enabled) { for (i = 0; i < num_sccs; ++i) { @@ -920,21 +1337,7 @@ processing_finish (int generation) } } - /* free callback data */ - - for (i = 0; i < num_sccs; ++i) { - sgen_free_internal_dynamic (api_sccs [i], - sizeof (MonoGCBridgeSCC) + sizeof (MonoObject*) * api_sccs [i]->num_objs, - INTERNAL_MEM_BRIDGE_DATA); - } - sgen_free_internal_dynamic (api_sccs, sizeof (MonoGCBridgeSCC*) * num_sccs, INTERNAL_MEM_BRIDGE_DATA); - - sgen_free_internal_dynamic (api_xrefs, sizeof (MonoGCBridgeXRef) * num_xrefs, INTERNAL_MEM_BRIDGE_DATA); - - SGEN_TV_GETTIME (atv); - step_8 = SGEN_TV_ELAPSED (btv, atv); - - mono_trace (G_LOG_LEVEL_INFO, MONO_TRACE_GC, "GC_BRIDGE num-objects %d num_hash_entries %d sccs size %d init %.2fms df1 %.2fms sort %.2fms dfs2 %.2fms setup-cb %.2fms free-data %.2fms user-cb %.2fms clenanup %.2fms links %d/%d/%d/%d dfs passes %d/%d", + mono_trace (G_LOG_LEVEL_INFO, MONO_TRACE_GC, "GC_NEW_BRIDGE num-objects %d num_hash_entries %d sccs size %d init %.2fms df1 %.2fms sort %.2fms dfs2 %.2fms setup-cb %.2fms free-data %.2fms links %d/%d/%d/%d dfs passes %d/%d ignored %d", num_registered_bridges, hash_table_size, dyn_array_scc_size (&sccs), step_1 / 10000.0f, step_2 / 10000.0f, @@ -942,16 +1345,12 @@ processing_finish (int generation) step_4 / 10000.0f, step_5 / 10000.0f, step_6 / 10000.0f, - step_7 / 10000.0f, - step_8 / 10000.f, fist_pass_links, second_pass_links, sccs_links, max_sccs_links, - dfs1_passes, dfs2_passes); + dfs1_passes, dfs2_passes, ignored_objects); step_1 = 0; /* We must cleanup since this value is used as an accumulator. */ fist_pass_links = second_pass_links = sccs_links = max_sccs_links = 0; - dfs1_passes = dfs2_passes = 0; - - bridge_processing_in_progress = FALSE; + dfs1_passes = dfs2_passes = ignored_objects = 0; } static void @@ -973,7 +1372,7 @@ describe_pointer (MonoObject *obj) printf ("Bridge hash table entry %p:\n", entry); printf (" is bridge: %d\n", (int)entry->is_bridge); - printf (" is visited: %d\n", (int)entry->is_visited); + printf (" is visited: %d\n", (int)entry->v.dfs1.is_visited); } void @@ -981,11 +1380,15 @@ sgen_new_bridge_init (SgenBridgeProcessor *collector) { collector->reset_data = reset_data; collector->processing_stw_step = processing_stw_step; - collector->processing_finish = processing_finish; + collector->processing_build_callback_data = processing_build_callback_data; + collector->processing_after_callback = processing_after_callback; collector->class_kind = class_kind; collector->register_finalized_object = register_finalized_object; collector->describe_pointer = describe_pointer; collector->enable_accounting = enable_accounting; + collector->set_dump_prefix = set_dump_prefix; + + bridge_processor = collector; } #endif