assigned TLABs and if that's more than some percentage of the
nursery size, reduce the TLAB size.
+ *) Explore placing unreachable objects on unused nursery memory.
+ Instead of memset'ng a region to zero, place an int[] covering it.
+ A good place to start is add_nursery_frag. The tricky thing here is
+ placing those objects atomically outside of a collection.
+
+
*/
#include "config.h"
#ifdef HAVE_SGEN_GC
#endif
#ifdef HEAVY_STATISTICS
-static long stat_objects_alloced = 0;
-static long stat_bytes_alloced = 0;
-static long stat_objects_alloced_degraded = 0;
-static long stat_bytes_alloced_degraded = 0;
-static long stat_bytes_alloced_los = 0;
-static long stat_copy_object_called_nursery = 0;
-static long stat_objects_copied_nursery = 0;
-static long stat_copy_object_called_major = 0;
-static long stat_objects_copied_major = 0;
-
-static long stat_copy_object_failed_from_space = 0;
-static long stat_copy_object_failed_forwarded = 0;
-static long stat_copy_object_failed_pinned = 0;
-static long stat_copy_object_failed_large_pinned = 0;
-static long stat_copy_object_failed_to_space = 0;
-
-static long stat_store_remsets = 0;
-static long stat_store_remsets_unique = 0;
-static long stat_saved_remsets_1 = 0;
-static long stat_saved_remsets_2 = 0;
-static long stat_global_remsets_added = 0;
-static long stat_global_remsets_processed = 0;
-
-static long num_copy_object_called = 0;
-static long num_objects_copied = 0;
+static long long stat_objects_alloced = 0;
+static long long stat_bytes_alloced = 0;
+static long long stat_objects_alloced_degraded = 0;
+static long long stat_bytes_alloced_degraded = 0;
+static long long stat_bytes_alloced_los = 0;
+
+static long long stat_copy_object_called_nursery = 0;
+static long long stat_objects_copied_nursery = 0;
+static long long stat_copy_object_called_major = 0;
+static long long stat_objects_copied_major = 0;
+
+static long long stat_scan_object_called_nursery = 0;
+static long long stat_scan_object_called_major = 0;
+
+static long long stat_nursery_copy_object_failed_from_space = 0;
+static long long stat_nursery_copy_object_failed_forwarded = 0;
+static long long stat_nursery_copy_object_failed_pinned = 0;
+
+static long long stat_store_remsets = 0;
+static long long stat_store_remsets_unique = 0;
+static long long stat_saved_remsets_1 = 0;
+static long long stat_saved_remsets_2 = 0;
+static long long stat_global_remsets_added = 0;
+static long long stat_global_remsets_readded = 0;
+static long long stat_global_remsets_processed = 0;
+static long long stat_global_remsets_discarded = 0;
+
+static long long stat_wasted_fragments_used = 0;
+static long long stat_wasted_fragments_bytes = 0;
static int stat_wbarrier_set_field = 0;
static int stat_wbarrier_set_arrayref = 0;
static int stat_wbarrier_object_copy = 0;
#endif
-static long time_minor_pre_collection_fragment_clear = 0;
-static long time_minor_pinning = 0;
-static long time_minor_scan_remsets = 0;
-static long time_minor_scan_pinned = 0;
-static long time_minor_scan_registered_roots = 0;
-static long time_minor_scan_thread_data = 0;
-static long time_minor_finish_gray_stack = 0;
-static long time_minor_fragment_creation = 0;
-
-static long time_major_pre_collection_fragment_clear = 0;
-static long time_major_pinning = 0;
-static long time_major_scan_pinned = 0;
-static long time_major_scan_registered_roots = 0;
-static long time_major_scan_thread_data = 0;
-static long time_major_scan_alloc_pinned = 0;
-static long time_major_scan_finalized = 0;
-static long time_major_scan_big_objects = 0;
-static long time_major_finish_gray_stack = 0;
-static long time_major_sweep = 0;
-static long time_major_fragment_creation = 0;
-
-static long pinned_chunk_bytes_alloced = 0;
-static long large_internal_bytes_alloced = 0;
-
+static long long time_minor_pre_collection_fragment_clear = 0;
+static long long time_minor_pinning = 0;
+static long long time_minor_scan_remsets = 0;
+static long long time_minor_scan_pinned = 0;
+static long long time_minor_scan_registered_roots = 0;
+static long long time_minor_scan_thread_data = 0;
+static long long time_minor_finish_gray_stack = 0;
+static long long time_minor_fragment_creation = 0;
+
+static long long time_major_pre_collection_fragment_clear = 0;
+static long long time_major_pinning = 0;
+static long long time_major_scan_pinned = 0;
+static long long time_major_scan_registered_roots = 0;
+static long long time_major_scan_thread_data = 0;
+static long long time_major_scan_alloc_pinned = 0;
+static long long time_major_scan_finalized = 0;
+static long long time_major_scan_big_objects = 0;
+static long long time_major_finish_gray_stack = 0;
+static long long time_major_sweep = 0;
+static long long time_major_fragment_creation = 0;
+
+static long long pinned_chunk_bytes_alloced = 0;
+static long long large_internal_bytes_alloced = 0;
+
+/* Keep in sync with internal_mem_names in dump_heap()! */
enum {
INTERNAL_MEM_PIN_QUEUE,
INTERNAL_MEM_FRAGMENT,
INTERNAL_MEM_REMSET,
INTERNAL_MEM_GRAY_QUEUE,
INTERNAL_MEM_STORE_REMSET,
+ INTERNAL_MEM_MS_TABLES,
+ INTERNAL_MEM_MS_BLOCK_INFO,
+ INTERNAL_MEM_EPHEMERON_LINK,
INTERNAL_MEM_MAX
};
struct _LOSObject {
LOSObject *next;
mword size; /* this is the object size */
- int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
guint16 role;
- guint16 scanned;
+ int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
char data [MONO_ZERO_LEN_ARRAY];
};
REMSET_LOCATION, /* just a pointer to the exact location */
REMSET_RANGE, /* range of pointer fields */
REMSET_OBJECT, /* mark all the object for scanning */
- REMSET_OTHER, /* all others */
+ REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
REMSET_TYPE_MASK = 0x3
};
-/* Subtypes of REMSET_OTHER */
-enum {
- REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
- REMSET_ROOT_LOCATION, /* a location inside a root */
-};
-
#ifdef HAVE_KW_THREAD
static __thread RememberedSet *remembered_set MONO_TLS_FAST;
#endif
static RememberedSet *freed_thread_remsets;
static GenericStoreRememberedSet *generic_store_remsets = NULL;
+/*A two slots cache for recently inserted remsets */
+static gpointer global_remset_cache [2];
+
/* FIXME: later choose a size that takes into account the RememberedSet struct
* and doesn't waste any alloc paddin space.
*/
{
MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
if (klass == mono_defaults.string_class) {
- return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
+ return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
} else if (klass->rank) {
MonoArray *array = (MonoArray*)o;
- size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length (array);
+ size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
if (G_UNLIKELY (array->bounds)) {
size += sizeof (mono_array_size_t) - 1;
size &= ~(sizeof (mono_array_size_t) - 1);
int num_links;
};
+typedef struct _EphemeronLinkNode EphemeronLinkNode;
+
+struct _EphemeronLinkNode {
+ EphemeronLinkNode *next;
+ char *array;
+};
+
+typedef struct {
+ void *key;
+ void *value;
+} Ephemeron;
+
#define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
static DisappearingLinkHashTable minor_disappearing_link_hash;
static DisappearingLinkHashTable major_disappearing_link_hash;
+static EphemeronLinkNode *ephemeron_list;
+
static int num_ready_finalizers = 0;
static int no_finalize = 0;
*/
static guint32 tlab_size = (1024 * 4);
+/*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
+#define MAX_NURSERY_TLAB_WASTE 512
+
/* fragments that are free and ready to be used for allocation */
static Fragment *nursery_fragments = NULL;
/* freeelist of fragment structures */
/* Functions supplied by the runtime to be called by the GC */
static MonoGCCallbacks gc_callbacks;
-#define ALLOC_ALIGN 8
+#define ALLOC_ALIGN 8
+#define ALLOC_ALIGN_BITS 3
+
+#define MOVED_OBJECTS_NUM 64
+static void *moved_objects [MOVED_OBJECTS_NUM];
+static int moved_objects_idx = 0;
/*
* ######################################################################
return (void*)p;
}
-typedef void (*CopyOrMarkObjectFunc) (void**, char*, char*);
-typedef char* (*ScanObjectFunc) (char*, char*, char*);
+typedef void (*CopyOrMarkObjectFunc) (void**);
+typedef char* (*ScanObjectFunc) (char*);
/* forward declarations */
static void* get_internal_mem (size_t size, int type);
static void free_internal_mem (void *addr, int type);
static void* get_os_memory (size_t size, int activate);
-static void* get_os_memory_aligned (mword size, gboolean activate);
+static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
static void free_os_memory (void *addr, size_t size);
static G_GNUC_UNUSED void report_internal_mem_usage (void);
static int stop_world (void);
static int restart_world (void);
-static void add_to_global_remset (gpointer ptr, gboolean root);
+static void add_to_global_remset (gpointer ptr);
static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
static void scan_from_remsets (void *start_nursery, void *end_nursery);
static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type);
-static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, char *start, char *end);
-static int scan_needed_big_objects (ScanObjectFunc scan_func, char *start_addr, char *end_addr);
+static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list);
static void find_pinning_ref_from_thread (char *obj, size_t size);
static void update_current_thread_stack (void *start);
static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
static void null_links_for_domain (MonoDomain *domain, int generation);
static gboolean search_fragment_for_size (size_t size);
+static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
static void build_nursery_fragments (int start_pin, int end_pin);
static void clear_nursery_fragments (char *next);
static void pin_from_roots (void *start_nursery, void *end_nursery);
static void clear_tlabs (void);
typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
-static char* scan_object (char *start, char* from_start, char* from_end);
-static void copy_object (void **obj_slot, char *from_space_start, char *from_space_end);
-static void scan_from_pinned_objects (char *addr_start, char *addr_end);
-static void scan_pinned_objects_in_section (GCMemSection *section, ScanObjectFunc scan_func, char *heap_start, char *heap_end);
-static void scan_pinned_objects_in_nursery (ScanObjectFunc scan_func, char *heap_start, char *heap_end);
+static char* scan_object (char *start);
+static char* major_scan_object (char *start);
+static void* copy_object_no_checks (void *obj);
+static void copy_object (void **obj_slot);
static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
static PinnedChunk* alloc_pinned_chunk (void);
static void free_large_object (LOSObject *obj);
static void sort_addresses (void **array, int size);
+static void drain_gray_stack (void);
static void finish_gray_stack (char *start_addr, char *end_addr, int generation);
static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
static void check_section_scan_starts (GCMemSection *section);
static void check_scan_starts (void);
static void check_for_xdomain_refs (void);
+static void dump_occupied (char *start, char *end, char *section_start);
static void dump_section (GCMemSection *section, const char *type);
static void dump_heap (const char *type, int num, const char *reason);
static void commit_stats (int generation);
static void init_stats (void);
+static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end);
+static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end);
+static void null_ephemerons_for_domain (MonoDomain *domain);
+
//#define BINARY_PROTOCOL
#include "sgen-protocol.c"
#include "sgen-pinning.c"
/* helper macros to scan and traverse objects, macros because we resue them in many functions */
#define STRING_SIZE(size,str) do { \
- (size) = sizeof (MonoString) + 2 * mono_string_length ((MonoString*)(str)) + 2; \
+ (size) = sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*)(str)) + 2; \
(size) += (ALLOC_ALIGN - 1); \
(size) &= ~(ALLOC_ALIGN - 1); \
} while (0)
int mbwords = (*mbitmap_data++) - 1; \
int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
- char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
+ char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
if (0) { \
MonoObject *myobj = (MonoObject*)start; \
g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
int etype = (vt)->desc & 0xc000; \
if (etype == (DESC_TYPE_V_REFS << 14)) { \
void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
- void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
+ void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
/* Note: this code can handle also arrays of struct with only references in them */ \
while (p < end_refs) { \
HANDLE_PTR (p, (obj)); \
int offset = ((vt)->desc >> 16) & 0xff; \
int num_refs = ((vt)->desc >> 24) & 0xff; \
char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
- char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
+ char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
while (e_start < e_end) { \
void **p = (void**)e_start; \
int i; \
} \
} else if (etype == DESC_TYPE_V_BITMAP << 14) { \
char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
- char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
+ char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
while (e_start < e_end) { \
void **p = (void**)e_start; \
gsize _bmap = (vt)->desc >> 16; \
} while (0)
#include "sgen-major-copying.c"
+//#include "sgen-marksweep.c"
static gboolean
is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
scan_area_with_callback (nursery_section->data, nursery_section->end_data,
(IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
+ /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
+ to memory returned to the OS.*/
+ null_ephemerons_for_domain (domain);
+
+ for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
+ null_links_for_domain (domain, i);
+
/* We need two passes over major and large objects because
freeing such objects might give their memory back to the OS
(in the case of large objects) or obliterate its vtable
major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
- for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
- null_links_for_domain (domain, i);
-
UNLOCK_GC;
}
+static void
+global_remset_cache_clear (void)
+{
+ memset (global_remset_cache, 0, sizeof (global_remset_cache));
+}
+
+/*
+ * Tries to check if a given remset location was already added to the global remset.
+ * It can
+ *
+ * A 2 entry, LRU cache of recently saw location remsets.
+ *
+ * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
+ *
+ * Returns TRUE is the element was added..
+ */
+static gboolean
+global_remset_location_was_not_added (gpointer ptr)
+{
+
+ gpointer first = global_remset_cache [0], second;
+ if (first == ptr) {
+ HEAVY_STAT (++stat_global_remsets_discarded);
+ return FALSE;
+ }
+
+ second = global_remset_cache [1];
+
+ if (second == ptr) {
+ /*Move the second to the front*/
+ global_remset_cache [0] = second;
+ global_remset_cache [1] = first;
+
+ HEAVY_STAT (++stat_global_remsets_discarded);
+ return FALSE;
+ }
+
+ global_remset_cache [0] = second;
+ global_remset_cache [1] = ptr;
+ return TRUE;
+}
+
/*
* add_to_global_remset:
*
* a minor collection. This can happen if the objects they point to are pinned.
*/
static void
-add_to_global_remset (gpointer ptr, gboolean root)
+add_to_global_remset (gpointer ptr)
{
RememberedSet *rs;
+ g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
+
+ if (!global_remset_location_was_not_added (ptr))
+ return;
+
DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
- g_assert (!root);
- g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
-
HEAVY_STAT (++stat_global_remsets_added);
/*
* To avoid uncontrolled growth of the global remset, only add each pointer once.
*/
if (global_remset->store_next + 3 < global_remset->end_set) {
- if (root) {
- *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
- *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
- } else {
- *(global_remset->store_next++) = (mword)ptr;
- }
+ *(global_remset->store_next++) = (mword)ptr;
return;
}
rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
rs->next = global_remset;
global_remset = rs;
- if (root) {
- *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
- *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
- } else {
- *(global_remset->store_next++) = (mword)ptr;
- }
+ *(global_remset->store_next++) = (mword)ptr;
{
int global_rs_size = 0;
}
}
-#define MOVED_OBJECTS_NUM 64
-static void *moved_objects [MOVED_OBJECTS_NUM];
-static int moved_objects_idx = 0;
-
/*
- * This is how the copying happens from the nursery to the old generation.
- * We assume that at this time all the pinned objects have been identified and
- * marked as such.
- * We run scan_object() for each pinned object so that each referenced
- * objects if possible are copied. The new gray objects created can have
- * scan_object() run on them right away, too.
- * Then we run copy_object() for the precisely tracked roots. At this point
- * all the roots are either gray or black. We run scan_object() on the gray
- * objects until no more gray objects are created.
- * At the end of the process we walk again the pinned list and we unmark
- * the pinned flag. As we go we also create the list of free space for use
- * in the next allocation runs.
- *
- * We need to remember objects from the old generation that point to the new one
- * (or just addresses?).
- *
- * copy_object could be made into a macro once debugged (use inline for now).
+ * FIXME: allocate before calling this function and pass the
+ * destination address.
*/
-
-static void __attribute__((noinline))
-copy_object (void **obj_slot, char *from_space_start, char *from_space_end)
+static void*
+copy_object_no_checks (void *obj)
{
static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
- char *destination;
- char *forwarded;
- char *obj = *obj_slot;
mword objsize;
- MonoVTable *vt;
-
- HEAVY_STAT (++num_copy_object_called);
-
- /* FIXME: For the minor collector, only the ptr_in_nursery()
- check is necessary. The rest must only be implemented in
- major_copy_or_mark_object(). */
-
- if (!(obj >= from_space_start && obj < from_space_end)) {
- DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
- obj, from_space_start, from_space_end));
- HEAVY_STAT (++stat_copy_object_failed_from_space);
- return;
- }
-
- DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
-
- /*
- * obj must belong to one of:
- *
- * 1. the nursery
- * 2. the LOS
- * 3. a pinned chunk
- * 4. a non-to-space section of the major heap
- * 5. a to-space section of the major heap
- *
- * In addition, objects in 1, 2 and 4 might also be pinned.
- * Objects in 1 and 4 might be forwarded.
- *
- * Before we can copy the object we must make sure that we are
- * allowed to, i.e. that the object not pinned, not already
- * forwarded and doesn't belong to the LOS, a pinned chunk, or
- * a to-space section.
- *
- * We are usually called for to-space objects (5) when we have
- * two remset entries for the same reference. The first entry
- * copies the object and updates the reference and the second
- * calls us with the updated reference that points into
- * to-space. There might also be other circumstances where we
- * get to-space objects.
- */
-
- if ((forwarded = object_is_forwarded (obj))) {
- DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
- DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
- HEAVY_STAT (++stat_copy_object_failed_forwarded);
- *obj_slot = forwarded;
- return;
- }
- if (object_is_pinned (obj)) {
- DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
- DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
- HEAVY_STAT (++stat_copy_object_failed_pinned);
- return;
- }
+ char *destination;
+ MonoVTable *vt = ((MonoObject*)obj)->vtable;
+ gboolean has_references = vt->klass->has_references;
objsize = safe_object_get_size ((MonoObject*)obj);
objsize += ALLOC_ALIGN - 1;
objsize &= ~(ALLOC_ALIGN - 1);
- if (ptr_in_nursery (obj))
- goto copy;
-
- /*
- * At this point we know obj is not pinned, not forwarded and
- * belongs to 2, 3, 4, or 5.
- *
- * LOS object (2) are simple, at least until we always follow
- * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
- * and return it. At the end of major collections, we walk
- * the los list and if the object is pinned, it is marked,
- * otherwise it can be freed.
- *
- * Pinned chunks (3) and major heap sections (4, 5) both
- * reside in blocks, which are always aligned, so once we've
- * eliminated LOS objects, we can just access the block and
- * see whether it's a pinned chunk or a major heap section.
- */
- if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
- DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
- binary_protocol_pin (obj, (gpointer)LOAD_VTABLE (obj), safe_object_get_size ((MonoObject*)obj));
- pin_object (obj);
- HEAVY_STAT (++stat_copy_object_failed_large_pinned);
- return;
- }
-
- /*
- * Now we know the object is in a major heap section. All we
- * need to do is check whether it's already in to-space (5) or
- * not (4).
- */
- if (MAJOR_OBJ_IS_IN_TO_SPACE (obj)) {
- DEBUG (9, g_assert (objsize <= MAX_SMALL_OBJ_SIZE));
- DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
- HEAVY_STAT (++stat_copy_object_failed_to_space);
- return;
- }
-
- copy:
- MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize);
+ DEBUG (9, g_assert (vt->klass->inited));
+ MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", destination, ((MonoObject*)obj)->vtable->klass->name, objsize));
binary_protocol_copy (obj, destination, ((MonoObject*)obj)->vtable, objsize);
- HEAVY_STAT (++num_objects_copied);
-
if (objsize <= sizeof (gpointer) * 8) {
mword *dest = (mword*)destination;
goto *copy_labels [objsize / sizeof (gpointer)];
#endif
}
/* adjust array->bounds */
- vt = ((MonoObject*)obj)->vtable;
DEBUG (9, g_assert (vt->gc_descr));
if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
MonoArray *array = (MonoArray*)destination;
moved_objects [moved_objects_idx++] = destination;
}
obj = destination;
- DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
- GRAY_OBJECT_ENQUEUE (obj);
- *obj_slot = obj;
+ if (has_references) {
+ DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
+ GRAY_OBJECT_ENQUEUE (obj);
+ }
+ return obj;
+}
+
+/*
+ * This is how the copying happens from the nursery to the old generation.
+ * We assume that at this time all the pinned objects have been identified and
+ * marked as such.
+ * We run scan_object() for each pinned object so that each referenced
+ * objects if possible are copied. The new gray objects created can have
+ * scan_object() run on them right away, too.
+ * Then we run copy_object() for the precisely tracked roots. At this point
+ * all the roots are either gray or black. We run scan_object() on the gray
+ * objects until no more gray objects are created.
+ * At the end of the process we walk again the pinned list and we unmark
+ * the pinned flag. As we go we also create the list of free space for use
+ * in the next allocation runs.
+ *
+ * We need to remember objects from the old generation that point to the new one
+ * (or just addresses?).
+ *
+ * copy_object could be made into a macro once debugged (use inline for now).
+ */
+
+static void __attribute__((noinline))
+copy_object (void **obj_slot)
+{
+ char *forwarded;
+ char *obj = *obj_slot;
+
+ DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
+
+ HEAVY_STAT (++stat_copy_object_called_nursery);
+
+ if (!ptr_in_nursery (obj)) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
+ return;
+ }
+
+ DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
+
+ /*
+ * Before we can copy the object we must make sure that we are
+ * allowed to, i.e. that the object not pinned or not already
+ * forwarded.
+ */
+
+ if ((forwarded = object_is_forwarded (obj))) {
+ DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
+ *obj_slot = forwarded;
+ return;
+ }
+ if (object_is_pinned (obj)) {
+ DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
+ return;
+ }
+
+ HEAVY_STAT (++stat_objects_copied_nursery);
+
+ *obj_slot = copy_object_no_checks (obj);
}
#undef HANDLE_PTR
void *__old = *(ptr); \
void *__copy; \
if (__old) { \
- copy_object ((ptr), from_start, from_end); \
+ copy_object ((ptr)); \
__copy = *(ptr); \
DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
- add_to_global_remset ((ptr), FALSE); \
+ add_to_global_remset ((ptr)); \
} \
} while (0)
* Returns a pointer to the end of the object.
*/
static char*
-scan_object (char *start, char* from_start, char* from_end)
+scan_object (char *start)
{
#include "sgen-scan-object.h"
- return start;
-}
-
-static void
-scan_pinned_objects_in_section (GCMemSection *section, ScanObjectFunc scan_func, char *heap_start, char *heap_end)
-{
- int i;
- for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
- DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
- i, pin_queue [i], safe_name (pin_queue [i])));
- scan_func (pin_queue [i], heap_start, heap_end);
- }
-}
+ HEAVY_STAT (++stat_scan_object_called_nursery);
-static void
-scan_pinned_objects_in_nursery (ScanObjectFunc scan_func, char *heap_start, char *heap_end)
-{
- scan_pinned_objects_in_section (nursery_section, scan_func, heap_start, heap_end);
-}
-
-/*
- * drain_gray_stack:
- *
- * Scan objects in the gray stack until the stack is empty. This should be called
- * frequently after each object is copied, to achieve better locality and cache
- * usage.
- */
-static void inline
-drain_gray_stack (char *start_addr, char *end_addr)
-{
- char *obj;
-
- if (current_collection_generation == GENERATION_NURSERY) {
- for (;;) {
- GRAY_OBJECT_DEQUEUE (obj);
- if (!obj)
- break;
- DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
- scan_object (obj, start_addr, end_addr);
- }
- } else {
- for (;;) {
- GRAY_OBJECT_DEQUEUE (obj);
- if (!obj)
- break;
- DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
- major_scan_object (obj, start_addr, end_addr);
- }
- }
+ return start;
}
/*
return NULL;
}
+#undef HANDLE_PTR
+#define HANDLE_PTR(ptr,obj) do { \
+ void *__old = *(ptr); \
+ void *__copy; \
+ if (__old) { \
+ major_copy_or_mark_object ((ptr)); \
+ __copy = *(ptr); \
+ DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
+ if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
+ add_to_global_remset ((ptr)); \
+ } \
+ } while (0)
+
+static char*
+major_scan_object (char *start)
+{
+#include "sgen-scan-object.h"
+
+ HEAVY_STAT (++stat_scan_object_called_major);
+
+ return start;
+}
+
+/*
+ * drain_gray_stack:
+ *
+ * Scan objects in the gray stack until the stack is empty. This should be called
+ * frequently after each object is copied, to achieve better locality and cache
+ * usage.
+ */
+static void inline
+drain_gray_stack (void)
+{
+ char *obj;
+
+ if (current_collection_generation == GENERATION_NURSERY) {
+ for (;;) {
+ GRAY_OBJECT_DEQUEUE (obj);
+ if (!obj)
+ break;
+ DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
+ scan_object (obj);
+ }
+ } else {
+ for (;;) {
+ GRAY_OBJECT_DEQUEUE (obj);
+ if (!obj)
+ break;
+ DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
+ major_scan_object (obj);
+ }
+ }
+}
+
/*
* Addresses from start to end are already sorted. This function finds
* the object header for each address and pins the object. The
DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
pin_object (search_start);
+ GRAY_OBJECT_ENQUEUE (search_start);
if (heap_dump_file)
pin_stats_register_object (search_start, last_obj_size);
definitely_pinned [count] = search_start;
return gap;
}
-#if 0
-static int
-compare_addr (const void *a, const void *b)
-{
- return *(const void **)a - *(const void **)b;
-}
-#endif
-
-/* sort the addresses in array in increasing order */
+/* Sort the addresses in array in increasing order.
+ * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
+ */
static void
sort_addresses (void **array, int size)
{
- /*
- * qsort is slower as predicted.
- * qsort (array, size, sizeof (gpointer), compare_addr);
- * return;
- */
- int gap = size;
- int swapped, end;
- while (TRUE) {
- int i;
- gap = new_gap (gap);
- swapped = FALSE;
- end = size - gap;
- for (i = 0; i < end; i++) {
- int j = i + gap;
- if (array [i] > array [j]) {
- void* val = array [i];
- array [i] = array [j];
- array [j] = val;
- swapped = TRUE;
- }
+ int i;
+ void *tmp;
+
+ for (i = 1; i < size; ++i) {
+ int child = i;
+ while (child > 0) {
+ int parent = (child - 1) / 2;
+
+ if (array [parent] >= array [child])
+ break;
+
+ tmp = array [parent];
+ array [parent] = array [child];
+ array [child] = tmp;
+
+ child = parent;
+ }
+ }
+
+ for (i = size - 1; i > 0; --i) {
+ int end, root;
+ tmp = array [i];
+ array [i] = array [0];
+ array [0] = tmp;
+
+ end = i - 1;
+ root = 0;
+
+ while (root * 2 + 1 <= end) {
+ int child = root * 2 + 1;
+
+ if (child < end && array [child] < array [child + 1])
+ ++child;
+ if (array [root] >= array [child])
+ break;
+
+ tmp = array [root];
+ array [root] = array [child];
+ array [child] = tmp;
+
+ root = child;
}
- if (gap == 1 && !swapped)
- break;
}
}
evacuate_pin_staging_area ();
}
-/* Copy function called from user defined mark functions */
-static char *user_copy_n_start;
-static char *user_copy_n_end;
-static CopyOrMarkObjectFunc user_copy_func;
-
-static void
-user_copy (void **addr)
-{
- user_copy_func (addr, user_copy_n_start, user_copy_n_end);
-}
-
/*
* The memory area from start_root to end_root contains pointers to objects.
* Their position is precisely described by @desc (this means that the pointer
desc >>= ROOT_DESC_TYPE_SHIFT;
while (desc) {
if ((desc & 1) && *start_root) {
- copy_func (start_root, n_start, n_end);
+ copy_func (start_root);
DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
- drain_gray_stack (n_start, n_end);
+ drain_gray_stack ();
}
desc >>= 1;
start_root++;
void **objptr = start_run;
while (bmap) {
if ((bmap & 1) && *objptr) {
- copy_func (objptr, n_start, n_end);
+ copy_func (objptr);
DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
- drain_gray_stack (n_start, n_end);
+ drain_gray_stack ();
}
bmap >>= 1;
++objptr;
}
case ROOT_DESC_USER: {
MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
-
- user_copy_n_start = n_start;
- user_copy_n_end = n_end;
- user_copy_func = copy_func;
- marker (start_root, user_copy);
+ marker (start_root, copy_func);
break;
}
case ROOT_DESC_RUN_LEN:
/* size must be a power of 2 */
static void*
-get_os_memory_aligned (mword size, gboolean activate)
+get_os_memory_aligned (mword size, mword alignment, gboolean activate)
{
/* Allocate twice the memory to be able to put the block on an aligned address */
- char *mem = get_os_memory (size * 2, activate);
+ char *mem = get_os_memory (size + alignment, activate);
char *aligned;
g_assert (mem);
- aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
- g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
+ aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
+ g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
if (aligned > mem)
free_os_memory (mem, aligned - mem);
- if (aligned + size < mem + size * 2)
- free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
+ if (aligned + size < mem + size + alignment)
+ free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
return aligned;
}
g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
alloc_size = nursery_size;
#ifdef ALIGN_NURSERY
- data = get_os_memory_aligned (alloc_size, TRUE);
+ data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
#else
data = get_os_memory (alloc_size, TRUE);
#endif
}
static void
-scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list, char *start, char *end) {
+scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list) {
FinalizeEntry *fin;
for (fin = list; fin; fin = fin->next) {
if (!fin->object)
continue;
DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
- copy_func (&fin->object, start, end);
+ copy_func (&fin->object);
}
}
fragment_total += frag_size;
} else {
/* Clear unused fragments, pinning depends on this */
+ /*TODO place an int[] here instead of the memset if size justify it*/
memset (frag_start, 0, frag_size);
}
}
-static int
-scan_needed_big_objects (ScanObjectFunc scan_func, char *start_addr, char *end_addr)
-{
- LOSObject *big_object;
- int count = 0;
- for (big_object = los_object_list; big_object; big_object = big_object->next) {
- if (!big_object->scanned && object_is_pinned (big_object->data)) {
- DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
- scan_func (big_object->data, start_addr, end_addr);
- drain_gray_stack (start_addr, end_addr);
- big_object->scanned = TRUE;
- count++;
- }
- }
- return count;
-}
-
static const char*
generation_name (int generation)
{
{
TV_DECLARE (atv);
TV_DECLARE (btv);
- int fin_ready, bigo_scanned_num;
+ int fin_ready;
+ int ephemeron_rounds = 0;
CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
- ScanObjectFunc scan_func = current_collection_generation == GENERATION_NURSERY ? scan_object : major_scan_object;
/*
* We copied all the reachable objects. Now it's the time to copy
* To achieve better cache locality and cache usage, we drain the gray stack
* frequently, after each object is copied, and just finish the work here.
*/
- drain_gray_stack (start_addr, end_addr);
+ drain_gray_stack ();
TV_GETTIME (atv);
DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
/* walk the finalization queue and move also the objects that need to be
* that are fin-ready. Speedup with a flag?
*/
do {
+ /*
+ * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
+ * before processing finalizable objects to avoid finalizing reachable values.
+ *
+ * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
+ * while they are been finalized.
+ */
+ int done_with_ephemerons = 0;
+ do {
+ done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr);
+ drain_gray_stack ();
+ ++ephemeron_rounds;
+ } while (!done_with_ephemerons);
+
fin_ready = num_ready_finalizers;
finalize_in_range (copy_func, start_addr, end_addr, generation);
if (generation == GENERATION_OLD)
- finalize_in_range (copy_object, nursery_start, nursery_real_end, GENERATION_NURSERY);
- bigo_scanned_num = scan_needed_big_objects (scan_func, start_addr, end_addr);
+ finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY);
/* drain the new stack that might have been created */
DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
- drain_gray_stack (start_addr, end_addr);
- } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
+ drain_gray_stack ();
+ } while (fin_ready != num_ready_finalizers);
+
+ /*
+ * Clear ephemeron pairs with unreachable keys.
+ * We pass the copy func so we can figure out if an array was promoted or not.
+ */
+ clear_unreachable_ephemerons (copy_func, start_addr, end_addr);
+
TV_GETTIME (btv);
- DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
+ DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
/*
* handle disappearing links
for (;;) {
null_link_in_range (copy_func, start_addr, end_addr, generation);
if (generation == GENERATION_OLD)
- null_link_in_range (copy_object, start_addr, end_addr, GENERATION_NURSERY);
+ null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY);
if (gray_object_queue_is_empty ())
break;
- drain_gray_stack (start_addr, end_addr);
+ drain_gray_stack ();
}
g_assert (gray_object_queue_is_empty ());
static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
"fin-table", "finalize-entry", "dislink-table",
"dislink", "roots-table", "root-record", "statistics",
- "remset", "gray-queue", "store-remset" };
+ "remset", "gray-queue", "store-remset", "marksweep-tables",
+ "marksweep-block-info", "ephemeron-link" };
ObjectList *list;
LOSObject *bigobj;
if (reason)
fprintf (heap_dump_file, " reason=\"%s\"", reason);
fprintf (heap_dump_file, ">\n");
- fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
- fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
+ fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
+ fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
for (i = 0; i < INTERNAL_MEM_MAX; ++i)
fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
dump_section (nursery_section, "nursery");
- major_dump_non_pinned_sections ();
+ major_dump_heap ();
fprintf (heap_dump_file, "<los>\n");
for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
{
static gboolean inited = FALSE;
-#ifdef HEAVY_STATISTICS
- num_copy_object_called = 0;
- num_objects_copied = 0;
-#endif
-
if (inited)
return;
mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
+
mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
- mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
- mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
- mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
- mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
- mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
+ mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
+ mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
+
+ mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
+ mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
+ mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
+
+ mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
+ mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
+ mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
+ mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
+
#endif
inited = TRUE;
}
-static void
-commit_stats (int generation)
-{
-#ifdef HEAVY_STATISTICS
- if (generation == GENERATION_NURSERY) {
- stat_copy_object_called_nursery += num_copy_object_called;
- stat_objects_copied_nursery += num_objects_copied;
- } else {
- g_assert (generation == GENERATION_OLD);
- stat_copy_object_called_major += num_copy_object_called;
- stat_objects_copied_major += num_objects_copied;
- }
-#endif
-}
-
/*
* Collect objects in the nursery. Returns whether to trigger a major
* collection.
num_minor_gcs++;
mono_stats.minor_gc_count ++;
+
+ global_remset_cache_clear ();
+
/* pin from pinned handles */
init_pinning ();
pin_from_roots (nursery_start, nursery_next);
DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
- if (consistency_check_at_minor_collection)
- check_consistency ();
+ if (consistency_check_at_minor_collection)
+ check_consistency ();
+
+ /*
+ * walk all the roots and copy the young objects to the old generation,
+ * starting from to_space
+ */
+
+ scan_from_remsets (nursery_start, nursery_next);
+ /* we don't have complete write barrier yet, so we scan all the old generation sections */
+ TV_GETTIME (btv);
+ time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
+ DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
+
+ drain_gray_stack ();
+
+ TV_GETTIME (atv);
+ time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
+ /* registered roots, this includes static fields */
+ scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL);
+ scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
+ TV_GETTIME (btv);
+ time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
+ /* thread data */
+ scan_thread_data (nursery_start, nursery_next, TRUE);
+ TV_GETTIME (atv);
+ time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
+ btv = atv;
+
+ finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
+ TV_GETTIME (atv);
+ time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
+
+ /* walk the pin_queue, build up the fragment list of free memory, unmark
+ * pinned objects as we go, memzero() the empty fragments so they are ready for the
+ * next allocations.
+ */
+ build_nursery_fragments (0, next_pin_slot);
+ TV_GETTIME (btv);
+ time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
+ DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (atv, btv), fragment_total));
+
+ major_finish_nursery_collection ();
+
+ TV_GETTIME (all_btv);
+ mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
+
+ if (heap_dump_file)
+ dump_heap ("minor", num_minor_gcs - 1, NULL);
+
+ /* prepare the pin queue for the next collection */
+ last_num_pinned = next_pin_slot;
+ next_pin_slot = 0;
+ if (fin_ready_list || critical_fin_list) {
+ DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
+ mono_gc_finalize_notify ();
+ }
+ pin_stats_reset ();
+
+ g_assert (gray_object_queue_is_empty ());
+
+ check_scan_starts ();
+
+ current_collection_generation = -1;
+
+ return major_need_major_collection ();
+}
+
+static void
+major_do_collection (const char *reason)
+{
+ LOSObject *bigobj, *prevbo;
+ TV_DECLARE (all_atv);
+ TV_DECLARE (all_btv);
+ TV_DECLARE (atv);
+ TV_DECLARE (btv);
+ /* FIXME: only use these values for the precise scan
+ * note that to_space pointers should be excluded anyway...
+ */
+ char *heap_start = NULL;
+ char *heap_end = (char*)-1;
+ int old_num_major_sections = num_major_sections;
+ int num_major_sections_saved, save_target, allowance_target;
+
+ //count_ref_nonref_objs ();
+ //consistency_check ();
+
+ init_stats ();
+ binary_protocol_collection (GENERATION_OLD);
+ check_scan_starts ();
+ gray_object_queue_init ();
+
+ degraded_mode = 0;
+ DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
+ num_major_gcs++;
+ mono_stats.major_gc_count ++;
+
+ /* world must be stopped already */
+ TV_GETTIME (all_atv);
+ TV_GETTIME (atv);
+
+ /* Pinning depends on this */
+ clear_nursery_fragments (nursery_next);
+
+ TV_GETTIME (btv);
+ time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
+
+ if (xdomain_checks)
+ check_for_xdomain_refs ();
+
+ nursery_section->next_data = nursery_real_end;
+ /* we should also coalesce scanning from sections close to each other
+ * and deal with pointers outside of the sections later.
+ */
+ /* The remsets are not useful for a major collection */
+ clear_remsets ();
+ global_remset_cache_clear ();
+
+ TV_GETTIME (atv);
+ init_pinning ();
+ DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
+ pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address);
+ optimize_pin_queue (0);
+
+ /*
+ * pin_queue now contains all candidate pointers, sorted and
+ * uniqued. We must do two passes now to figure out which
+ * objects are pinned.
+ *
+ * The first is to find within the pin_queue the area for each
+ * section. This requires that the pin_queue be sorted. We
+ * also process the LOS objects and pinned chunks here.
+ *
+ * The second, destructive, pass is to reduce the section
+ * areas to pointers to the actually pinned objects.
+ */
+ DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
+ /* first pass for the sections */
+ find_section_pin_queue_start_end (nursery_section);
+ major_find_pin_queue_start_ends ();
+ /* identify possible pointers to the insize of large objects */
+ DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
+ for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
+ int start, end;
+ find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
+ if (start != end) {
+ pin_object (bigobj->data);
+ /* FIXME: only enqueue if object has references */
+ GRAY_OBJECT_ENQUEUE (bigobj->data);
+ if (heap_dump_file)
+ pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
+ DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
+ }
+ }
+ /* second pass for the sections */
+ pin_objects_in_section (nursery_section);
+ major_pin_objects ();
+
+ TV_GETTIME (btv);
+ time_major_pinning += TV_ELAPSED_MS (atv, btv);
+ DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
+ DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
+
+ major_init_to_space ();
- /*
- * walk all the roots and copy the young objects to the old generation,
- * starting from to_space
- */
+ drain_gray_stack ();
- scan_from_remsets (nursery_start, nursery_next);
- /* we don't have complete write barrier yet, so we scan all the old generation sections */
+ TV_GETTIME (atv);
+ time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
+
+ /* registered roots, this includes static fields */
+ scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL);
+ scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER);
TV_GETTIME (btv);
- time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
- DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
+ time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
- /* the pinned objects are roots */
- scan_pinned_objects_in_nursery (scan_object, nursery_start, nursery_next);
+ /* Threads */
+ /* FIXME: This is the wrong place for this, because it does
+ pinning */
+ scan_thread_data (heap_start, heap_end, TRUE);
TV_GETTIME (atv);
- time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
- /* registered roots, this includes static fields */
- scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL);
- scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
+ time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
+
TV_GETTIME (btv);
- time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
- /* thread data */
- scan_thread_data (nursery_start, nursery_next, TRUE);
+ time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
+
+ /* scan the list of objects ready for finalization */
+ scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list);
+ scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list);
TV_GETTIME (atv);
- time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
- btv = atv;
+ time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
+ DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
- finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
+ TV_GETTIME (btv);
+ time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
+
+ /* all the objects in the heap */
+ finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
TV_GETTIME (atv);
- time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
+ time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
+
+ /* sweep the big objects list */
+ prevbo = NULL;
+ for (bigobj = los_object_list; bigobj;) {
+ if (object_is_pinned (bigobj->data)) {
+ unpin_object (bigobj->data);
+ } else {
+ LOSObject *to_free;
+ /* not referenced anywhere, so we can free it */
+ if (prevbo)
+ prevbo->next = bigobj->next;
+ else
+ los_object_list = bigobj->next;
+ to_free = bigobj;
+ bigobj = bigobj->next;
+ free_large_object (to_free);
+ continue;
+ }
+ prevbo = bigobj;
+ bigobj = bigobj->next;
+ }
+
+ major_sweep ();
+
+ TV_GETTIME (btv);
+ time_major_sweep += TV_ELAPSED_MS (atv, btv);
/* walk the pin_queue, build up the fragment list of free memory, unmark
* pinned objects as we go, memzero() the empty fragments so they are ready for the
* next allocations.
*/
- build_nursery_fragments (0, next_pin_slot);
- TV_GETTIME (btv);
- time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
- DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (atv, btv), fragment_total));
+ build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
- major_finish_nursery_collection ();
+ TV_GETTIME (atv);
+ time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
TV_GETTIME (all_btv);
- mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
+ mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
if (heap_dump_file)
- dump_heap ("minor", num_minor_gcs - 1, NULL);
+ dump_heap ("major", num_major_gcs - 1, reason);
/* prepare the pin queue for the next collection */
- last_num_pinned = next_pin_slot;
next_pin_slot = 0;
if (fin_ready_list || critical_fin_list) {
DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers));
g_assert (gray_object_queue_is_empty ());
- commit_stats (GENERATION_NURSERY);
+ num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
- check_scan_starts ();
+ save_target = num_major_sections / 2;
+ /*
+ * We aim to allow the allocation of as many sections as is
+ * necessary to reclaim save_target sections in the next
+ * collection. We assume the collection pattern won't change.
+ * In the last cycle, we had num_major_sections_saved for
+ * minor_collection_sections_alloced. Assuming things won't
+ * change, this must be the same ratio as save_target for
+ * allowance_target, i.e.
+ *
+ * num_major_sections_saved save_target
+ * --------------------------------- == ----------------
+ * minor_collection_sections_alloced allowance_target
+ *
+ * hence:
+ */
+ allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
- current_collection_generation = -1;
+ minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
- return major_need_major_collection ();
+ minor_collection_sections_alloced = 0;
+
+ check_scan_starts ();
+
+ //consistency_check ();
}
static void
int offset;
int size = PINNED_CHUNK_SIZE;
- chunk = get_os_memory_aligned (size, TRUE);
+ chunk = get_os_memory_aligned (size, size, TRUE);
chunk->block.role = MEMORY_ROLE_PINNED;
UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
alloc_size &= ~(pagesize - 1);
/* FIXME: handle OOM */
obj = get_os_memory (alloc_size, TRUE);
+ g_assert (!((mword)obj->data & (ALLOC_ALIGN - 1)));
obj->size = size;
vtslot = (void**)obj->data;
*vtslot = vtable;
return obj->data;
}
+static void
+setup_fragment (Fragment *frag, Fragment *prev, size_t size)
+{
+ /* remove from the list */
+ if (prev)
+ prev->next = frag->next;
+ else
+ nursery_fragments = frag->next;
+ nursery_next = frag->fragment_start;
+ nursery_frag_real_end = frag->fragment_end;
+
+ DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
+ frag->next = fragment_freelist;
+ fragment_freelist = frag;
+}
+
/* check if we have a suitable fragment in nursery_fragments to be able to allocate
* an object of size @size
* Return FALSE if not found (which means we need a collection)
prev = NULL;
for (frag = nursery_fragments; frag; frag = frag->next) {
if (size <= (frag->fragment_end - frag->fragment_start)) {
- /* remove from the list */
- if (prev)
- prev->next = frag->next;
- else
- nursery_fragments = frag->next;
- nursery_next = frag->fragment_start;
- nursery_frag_real_end = frag->fragment_end;
-
- DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
- frag->next = fragment_freelist;
- fragment_freelist = frag;
+ setup_fragment (frag, prev, size);
return TRUE;
}
prev = frag;
return FALSE;
}
+/*
+ * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
+ * This improves nursery usage.
+ */
+static int
+search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
+{
+ Fragment *frag, *prev, *min_prev;
+ DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
+
+ if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
+ /* Clear the remaining space, pinning depends on this */
+ memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
+
+ min_prev = GINT_TO_POINTER (-1);
+ prev = NULL;
+
+ for (frag = nursery_fragments; frag; frag = frag->next) {
+ int frag_size = frag->fragment_end - frag->fragment_start;
+ if (desired_size <= frag_size) {
+ setup_fragment (frag, prev, desired_size);
+ return desired_size;
+ }
+ if (minimum_size <= frag_size)
+ min_prev = prev;
+
+ prev = frag;
+ }
+
+ if (min_prev != GINT_TO_POINTER (-1)) {
+ int frag_size;
+ if (min_prev)
+ frag = min_prev->next;
+ else
+ frag = nursery_fragments;
+
+ frag_size = frag->fragment_end - frag->fragment_start;
+ HEAVY_STAT (++stat_wasted_fragments_used);
+ HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
+
+ setup_fragment (frag, min_prev, minimum_size);
+ return frag_size;
+ }
+
+ return 0;
+}
+
/*
* Provide a variant that takes just the vtable for small fixed-size objects.
* The aligned size is already computed and stored in vt->gc_descr.
/* FIXME: handle OOM */
void **p;
char *new_next;
- gboolean res;
TLAB_ACCESS_INIT;
HEAVY_STAT (++stat_objects_alloced);
return p;
}
+ /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
if (size > tlab_size) {
/* Allocate directly from the nursery */
if (nursery_next + size >= nursery_frag_real_end) {
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
memset (p, 0, size);
} else {
+ int alloc_size = tlab_size;
+ int available_in_nursery = nursery_frag_real_end - nursery_next;
if (TLAB_START)
DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
- if (nursery_next + tlab_size >= nursery_frag_real_end) {
- res = search_fragment_for_size (tlab_size);
- if (!res) {
- minor_collect_or_expand_inner (tlab_size);
- if (degraded_mode) {
- p = alloc_degraded (vtable, size);
- return p;
+ if (alloc_size >= available_in_nursery) {
+ if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
+ alloc_size = available_in_nursery;
+ } else {
+ alloc_size = search_fragment_for_size_range (tlab_size, size);
+ if (!alloc_size) {
+ alloc_size = tlab_size;
+ minor_collect_or_expand_inner (tlab_size);
+ if (degraded_mode) {
+ p = alloc_degraded (vtable, size);
+ return p;
+ }
}
}
}
/* Allocate a new TLAB from the current nursery fragment */
TLAB_START = nursery_next;
- nursery_next += tlab_size;
+ nursery_next += alloc_size;
TLAB_NEXT = TLAB_START;
- TLAB_REAL_END = TLAB_START + tlab_size;
- TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
+ TLAB_REAL_END = TLAB_START + alloc_size;
+ TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
- memset (TLAB_START, 0, tlab_size);
+ memset (TLAB_START, 0, alloc_size);
/* Allocate from the TLAB */
p = (void*)TLAB_NEXT;
/* large objects are always pinned anyway */
p = alloc_large_inner (vtable, size);
} else {
- p = major_alloc_small_pinned_obj (size);
+ DEBUG (9, g_assert (vtable->klass->inited));
+ p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
}
DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p, vtable, size);
if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
gboolean is_fin_ready = object_is_fin_ready (entry->object);
char *copy = entry->object;
- copy_func ((void**)©, start, end);
+ copy_func ((void**)©);
if (is_fin_ready) {
char *from;
FinalizeEntry *next;
}
}
+static int
+object_is_reachable (char *object, char *start, char *end)
+{
+ /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
+ if (object < start || object >= end)
+ return TRUE;
+ return !object_is_fin_ready (object) || major_is_object_live (object);
+}
+
+/* LOCKING: requires that the GC lock is held */
+static void
+null_ephemerons_for_domain (MonoDomain *domain)
+{
+ EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
+
+ while (current) {
+ MonoObject *object = (MonoObject*)current->array;
+
+ if (object && !object->vtable) {
+ EphemeronLinkNode *tmp = current;
+
+ if (prev)
+ prev->next = current->next;
+ else
+ ephemeron_list = current->next;
+
+ current = current->next;
+ free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
+ } else {
+ prev = current;
+ current = current->next;
+ }
+ }
+}
+
+/* LOCKING: requires that the GC lock is held */
+static void
+clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end)
+{
+ int was_in_nursery, was_promoted;
+ EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
+ MonoArray *array;
+ Ephemeron *cur, *array_end;
+ char *tombstone;
+
+ while (current) {
+ char *object = current->array;
+
+ if (!object_is_reachable (object, start, end)) {
+ EphemeronLinkNode *tmp = current;
+
+ DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
+
+ if (prev)
+ prev->next = current->next;
+ else
+ ephemeron_list = current->next;
+
+ current = current->next;
+ free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
+
+ continue;
+ }
+
+ was_in_nursery = ptr_in_nursery (object);
+ copy_func ((void**)&object);
+ current->array = object;
+
+ /*The array was promoted, add global remsets for key/values left behind in nursery.*/
+ was_promoted = was_in_nursery && !ptr_in_nursery (object);
+
+ DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
+
+ array = (MonoArray*)object;
+ cur = mono_array_addr (array, Ephemeron, 0);
+ array_end = cur + mono_array_length_fast (array);
+ tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
+
+ for (; cur < array_end; ++cur) {
+ char *key = (char*)cur->key;
+
+ if (!key || key == tombstone)
+ continue;
+
+ DEBUG (5, fprintf (gc_debug_file, "[%d] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
+ key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
+ cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
+
+ if (!object_is_reachable (key, start, end)) {
+ cur->key = tombstone;
+ cur->value = NULL;
+ continue;
+ }
+
+ if (was_promoted) {
+ if (ptr_in_nursery (key)) {/*key was not promoted*/
+ DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
+ add_to_global_remset (&cur->key);
+ }
+ if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
+ DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
+ add_to_global_remset (&cur->value);
+ }
+ }
+ }
+ prev = current;
+ current = current->next;
+ }
+}
+
+/* LOCKING: requires that the GC lock is held */
+static int
+mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end)
+{
+ int nothing_marked = 1;
+ EphemeronLinkNode *current = ephemeron_list;
+ MonoArray *array;
+ Ephemeron *cur, *array_end;
+ char *tombstone;
+
+ for (current = ephemeron_list; current; current = current->next) {
+ char *object = current->array;
+ DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
+
+ /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
+ if (object < start || object >= end)
+ continue;
+
+ /*It has to be alive*/
+ if (!object_is_reachable (object, start, end)) {
+ DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
+ continue;
+ }
+
+ copy_func ((void**)&object);
+
+ array = (MonoArray*)object;
+ cur = mono_array_addr (array, Ephemeron, 0);
+ array_end = cur + mono_array_length_fast (array);
+ tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
+
+ for (; cur < array_end; ++cur) {
+ char *key = cur->key;
+
+ if (!key || key == tombstone)
+ continue;
+
+ DEBUG (5, fprintf (gc_debug_file, "[%d] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
+ key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
+ cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
+
+ if (object_is_reachable (key, start, end)) {
+ char *value = cur->value;
+
+ copy_func ((void**)&cur->key);
+ if (value) {
+ if (!object_is_reachable (value, start, end))
+ nothing_marked = 0;
+ copy_func ((void**)&cur->value);
+ }
+ }
+ }
+ }
+
+ DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
+ return nothing_marked;
+}
+
/* LOCKING: requires that the GC lock is held */
static void
null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
continue;
} else {
char *copy = object;
- copy_func ((void**)©, start, end);
+ copy_func ((void**)©);
/* Update pointer if it's moved. If the object
* has been moved out of the nursery, we need to
prev = NULL;
for (entry = disappearing_link_hash [i]; entry; ) {
char *object = DISLINK_OBJECT (entry);
- /* FIXME: actually there should be no object
- left in the domain with a non-null vtable
- (provided we remove the Thread special
- case) */
- if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
+ if (object && !((MonoObject*)object)->vtable) {
DisappearingLink *next = entry->next;
if (prev)
mono_gc_scan_object (void *obj)
{
if (current_collection_generation == GENERATION_NURSERY)
- copy_object (&obj, scan_area_arg_start, scan_area_arg_end);
+ copy_object (&obj);
else
- major_copy_or_mark_object (&obj, scan_area_arg_end, scan_area_arg_end);
+ major_copy_or_mark_object (&obj);
return obj;
}
//__builtin_prefetch (ptr);
if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
gpointer old = *ptr;
- copy_object (ptr, start_nursery, end_nursery);
+ copy_object (ptr);
DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
if (old)
binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
* becomes part of the global remset, which can grow very large.
*/
DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
- add_to_global_remset (ptr, FALSE);
+ add_to_global_remset (ptr);
}
} else {
DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
return p + 2;
count = p [1];
while (count-- > 0) {
- copy_object (ptr, start_nursery, end_nursery);
+ copy_object (ptr);
DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
if (!global && *ptr >= start_nursery && *ptr < end_nursery)
- add_to_global_remset (ptr, FALSE);
+ add_to_global_remset (ptr);
++ptr;
}
return p + 2;
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
return p + 1;
- scan_object ((char*)ptr, start_nursery, end_nursery);
+ scan_object ((char*)ptr);
return p + 1;
- case REMSET_OTHER: {
+ case REMSET_VTYPE: {
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
-
- switch (p [1]) {
- case REMSET_VTYPE:
- if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
- return p + 4;
- desc = p [2];
- count = p [3];
- while (count-- > 0)
- ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
- return p + 4;
- case REMSET_ROOT_LOCATION:
- /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
- copy_object (ptr, start_nursery, end_nursery);
- DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
- if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
- /*
- * If the object is pinned, each reference to it from nonpinned objects
- * becomes part of the global remset, which can grow very large.
- */
- DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
- add_to_global_remset (ptr, TRUE);
- }
- return p + 2;
- default:
- g_assert_not_reached ();
- }
- break;
+ if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
+ return p + 3;
+ desc = p [1];
+ count = p [2];
+ while (count-- > 0)
+ ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
+ return p + 3;
}
default:
g_assert_not_reached ();
case REMSET_OBJECT:
p += 1;
break;
- case REMSET_OTHER:
- switch (p [1]) {
- case REMSET_VTYPE:
- p += 4;
- break;
- case REMSET_ROOT_LOCATION:
- p += 2;
- break;
- default:
- g_assert_not_reached ();
- }
+ case REMSET_VTYPE:
+ p += 3;
break;
default:
g_assert_not_reached ();
DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
store_pos = remset->data;
for (p = remset->data; p < remset->store_next; p = next_p) {
- mword ptr;
+ void **ptr = p [0];
+
+ /*Ignore previously processed remset.*/
+ if (!global_remset_location_was_not_added (ptr)) {
+ next_p = p + 1;
+ continue;
+ }
next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
* Clear global remsets of locations which no longer point to the
* nursery. Otherwise, they could grow indefinitely between major
* collections.
+ *
+ * Since all global remsets are location remsets, we don't need to unmask the pointer.
*/
- ptr = (p [0] & ~REMSET_TYPE_MASK);
- if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
- if (ptr_in_nursery (*(void**)ptr))
- *store_pos ++ = p [0];
- } else {
- g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
- g_assert (p [1] == REMSET_ROOT_LOCATION);
- if (ptr_in_nursery (*(void**)ptr)) {
- *store_pos ++ = p [0];
- *store_pos ++ = p [1];
- }
+ if (ptr_in_nursery (*ptr)) {
+ *store_pos ++ = p [0];
+ HEAVY_STAT (++stat_global_remsets_readded);
}
}
mono_gc_wbarrier_generic_nostore (ptr);
}
-void
-mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
-{
- RememberedSet *rs;
- TLAB_ACCESS_INIT;
- HEAVY_STAT (++stat_wbarrier_set_root);
- if (ptr_in_nursery (ptr))
- return;
- DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
-
- rs = REMEMBERED_SET;
- if (rs->store_next + 2 < rs->end_set) {
- *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
- *(void**)ptr = value;
- return;
- }
- rs = alloc_remset (rs->end_set - rs->data, (void*)1);
- rs->next = REMEMBERED_SET;
- REMEMBERED_SET = rs;
-#ifdef HAVE_KW_THREAD
- thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
-#endif
- *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
-
- *(void**)ptr = value;
-}
-
void
mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
{
DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
if (rs->store_next + 3 < rs->end_set) {
- *(rs->store_next++) = (mword)dest | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_VTYPE;
+ *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
*(rs->store_next++) = (mword)klass->gc_descr;
*(rs->store_next++) = (mword)count;
UNLOCK_GC;
#ifdef HAVE_KW_THREAD
thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
#endif
- *(rs->store_next++) = (mword)dest | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_VTYPE;
+ *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
*(rs->store_next++) = (mword)klass->gc_descr;
*(rs->store_next++) = (mword)count;
UNLOCK_GC;
if ((void**)addr >= ptr && (void**)addr < ptr + count)
*found = TRUE;
return p + 1;
- case REMSET_OTHER: {
- switch (p [1]) {
- case REMSET_VTYPE:
- ptr = (void**)(*p & ~REMSET_TYPE_MASK);
- desc = p [2];
- count = p [3];
-
- switch (desc & 0x7) {
- case DESC_TYPE_RUN_LENGTH:
- OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
- break;
- case DESC_TYPE_SMALL_BITMAP:
- OBJ_BITMAP_SIZE (skip_size, desc, start);
- break;
- default:
- // FIXME:
- g_assert_not_reached ();
- }
-
- /* The descriptor includes the size of MonoObject */
- skip_size -= sizeof (MonoObject);
- skip_size *= count;
- if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
- *found = TRUE;
+ case REMSET_VTYPE:
+ ptr = (void**)(*p & ~REMSET_TYPE_MASK);
+ desc = p [1];
+ count = p [2];
- return p + 4;
- case REMSET_ROOT_LOCATION:
- return p + 2;
+ switch (desc & 0x7) {
+ case DESC_TYPE_RUN_LENGTH:
+ OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
+ break;
+ case DESC_TYPE_SMALL_BITMAP:
+ OBJ_BITMAP_SIZE (skip_size, desc, start);
+ break;
default:
+ // FIXME:
g_assert_not_reached ();
}
- break;
- }
+
+ /* The descriptor includes the size of MonoObject */
+ skip_size -= sizeof (MonoObject);
+ skip_size *= count;
+ if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
+ *found = TRUE;
+
+ return p + 3;
default:
g_assert_not_reached ();
}
return (MonoObject*) REVEAL_POINTER (*link_addr);
}
+gboolean
+mono_gc_ephemeron_array_add (MonoObject *obj)
+{
+ EphemeronLinkNode *node;
+
+ LOCK_GC;
+
+ node = get_internal_mem (sizeof (EphemeronLinkNode), INTERNAL_MEM_EPHEMERON_LINK);
+ if (!node) {
+ UNLOCK_GC;
+ return FALSE;
+ }
+ node->array = (char*)obj;
+ node->next = ephemeron_list;
+ ephemeron_list = node;
+
+ DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
+
+ UNLOCK_GC;
+ return TRUE;
+}
+
void*
mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
{
projects / mono.git / blobdiff