sgen-gray.c \
sgen-gray.h \
sgen-major-copy-object.h \
+ sgen-minor-copy-object.h \
+ sgen-copy-object.h \
sgen-major-scan-object.h \
+ sgen-minor-scan-object.h \
sgen-protocol.h \
sgen-scan-object.h \
sgen-nursery-allocator.c \
--- /dev/null
+/*
+ * Copyright 2001-2003 Ximian, Inc
+ * Copyright 2003-2010 Novell, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+extern long long stat_copy_object_called_nursery;
+extern long long stat_objects_copied_nursery;
+
+extern long long stat_nursery_copy_object_failed_from_space;
+extern long long stat_nursery_copy_object_failed_forwarded;
+extern long long stat_nursery_copy_object_failed_pinned;
+
+extern long long stat_slots_allocated_in_vain;
+
+/*
+ * This function can be used even if the vtable of obj is not valid
+ * anymore, which is the case in the parallel collector.
+ */
+static void
+par_copy_object_no_checks (char *destination, MonoVTable *vt, void *obj, mword objsize, SgenGrayQueue *queue)
+{
+ static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
+
+ DEBUG (9, g_assert (vt->klass->inited));
+ DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
+ binary_protocol_copy (obj, destination, vt, objsize);
+
+ if (objsize <= sizeof (gpointer) * 8) {
+ mword *dest = (mword*)destination;
+ goto *copy_labels [objsize / sizeof (gpointer)];
+ LAB_8:
+ (dest) [7] = ((mword*)obj) [7];
+ LAB_7:
+ (dest) [6] = ((mword*)obj) [6];
+ LAB_6:
+ (dest) [5] = ((mword*)obj) [5];
+ LAB_5:
+ (dest) [4] = ((mword*)obj) [4];
+ LAB_4:
+ (dest) [3] = ((mword*)obj) [3];
+ LAB_3:
+ (dest) [2] = ((mword*)obj) [2];
+ LAB_2:
+ (dest) [1] = ((mword*)obj) [1];
+ LAB_1:
+ ;
+ LAB_0:
+ ;
+ } else {
+ /*can't trust memcpy doing word copies */
+ mono_gc_memmove (destination + sizeof (mword), (char*)obj + sizeof (mword), objsize - sizeof (mword));
+ }
+ /* adjust array->bounds */
+ DEBUG (9, g_assert (vt->gc_descr));
+ if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
+ MonoArray *array = (MonoArray*)destination;
+ array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
+ DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
+ }
+ if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES))
+ sgen_register_moved_object (obj, destination);
+ obj = destination;
+ if (queue) {
+ DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, sgen_safe_name (obj)));
+ GRAY_OBJECT_ENQUEUE (queue, obj);
+ }
+}
+
+static void*
+copy_object_no_checks (void *obj, SgenGrayQueue *queue)
+{
+ MonoVTable *vt = ((MonoObject*)obj)->vtable;
+ gboolean has_references = SGEN_VTABLE_HAS_REFERENCES (vt);
+ mword objsize = SGEN_ALIGN_UP (sgen_par_object_get_size (vt, (MonoObject*)obj));
+ char *destination = collector_serial_alloc_for_promotion (obj, objsize, has_references);
+
+ if (G_UNLIKELY (!destination)) {
+ collector_pin_object (obj, queue);
+ sgen_set_pinned_from_failed_allocation (objsize);
+ return obj;
+ }
+
+ *(MonoVTable**)destination = vt;
+ par_copy_object_no_checks (destination, vt, obj, objsize, has_references ? queue : NULL);
+
+ /* set the forwarding pointer */
+ SGEN_FORWARD_OBJECT (obj, destination);
+
+ return destination;
+}
+
+#ifdef GENERATE_COPY_FUNCTIONS
+
+/*
+ * This is how the copying happens from the nursery to the old generation.
+ * We assume that at this time all the pinned objects have been identified and
+ * marked as such.
+ * We run scan_object() for each pinned object so that each referenced
+ * objects if possible are copied. The new gray objects created can have
+ * scan_object() run on them right away, too.
+ * Then we run copy_object() for the precisely tracked roots. At this point
+ * all the roots are either gray or black. We run scan_object() on the gray
+ * objects until no more gray objects are created.
+ * At the end of the process we walk again the pinned list and we unmark
+ * the pinned flag. As we go we also create the list of free space for use
+ * in the next allocation runs.
+ *
+ * We need to remember objects from the old generation that point to the new one
+ * (or just addresses?).
+ *
+ * copy_object could be made into a macro once debugged (use inline for now).
+ */
+
+static void
+serial_copy_object (void **obj_slot, SgenGrayQueue *queue)
+{
+ char *forwarded;
+ char *obj = *obj_slot;
+
+ DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
+
+ HEAVY_STAT (++stat_copy_object_called_nursery);
+
+ if (!sgen_ptr_in_nursery (obj)) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
+ return;
+ }
+
+ DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
+
+ /*
+ * Before we can copy the object we must make sure that we are
+ * allowed to, i.e. that the object not pinned, not already
+ * forwarded or belongs to the nursery To Space.
+ */
+
+ if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj))) {
+ DEBUG (9, g_assert ((*(MonoVTable**)SGEN_LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
+ *obj_slot = forwarded;
+ return;
+ }
+ if (SGEN_OBJECT_IS_PINNED (obj)) {
+ DEBUG (9, g_assert (((MonoVTable*)SGEN_LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
+ return;
+ }
+
+ if (sgen_nursery_is_to_space (obj)) {
+ DEBUG (9, g_assert (((MonoVTable*)SGEN_LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (tospace, no change)\n"));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_to_space);
+ return;
+ }
+
+ HEAVY_STAT (++stat_objects_copied_nursery);
+
+ *obj_slot = copy_object_no_checks (obj, queue);
+}
+
+static void
+parallel_copy_object (void **obj_slot, SgenGrayQueue *queue)
+{
+ char *obj = *obj_slot;
+ mword vtable_word, objsize;
+ MonoVTable *vt;
+ void *destination;
+ gboolean has_references;
+
+ DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
+
+ HEAVY_STAT (++stat_copy_object_called_nursery);
+
+ if (!sgen_ptr_in_nursery (obj)) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
+ return;
+ }
+
+ vtable_word = *(mword*)obj;
+ vt = (MonoVTable*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
+
+ /*
+ * Before we can copy the object we must make sure that we are
+ * allowed to, i.e. that the object not pinned, not already
+ * forwarded and not in the nursery To Space.
+ */
+
+ if (vtable_word & SGEN_FORWARDED_BIT) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
+ *obj_slot = vt;
+ return;
+ }
+ if (vtable_word & SGEN_PINNED_BIT) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
+ return;
+ }
+
+ if (sgen_nursery_is_to_space (obj)) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_to_space);
+ return;
+ }
+
+ HEAVY_STAT (++stat_objects_copied_nursery);
+
+ objsize = SGEN_ALIGN_UP (sgen_par_object_get_size (vt, (MonoObject*)obj));
+ has_references = SGEN_VTABLE_HAS_REFERENCES (vt);
+
+ destination = collector_parallel_alloc_for_promotion (obj, objsize, has_references);
+
+ if (G_UNLIKELY (!destination)) {
+ sgen_parallel_pin_or_update (obj_slot, obj, vt, queue);
+ return;
+ }
+
+ *(MonoVTable**)destination = vt;
+
+ if (SGEN_CAS_PTR ((void*)obj, (void*)((mword)destination | SGEN_FORWARDED_BIT), vt) == vt) {
+ par_copy_object_no_checks (destination, vt, obj, objsize, has_references ? queue : NULL);
+ obj = destination;
+ *obj_slot = obj;
+ } else {
+ /* FIXME: unify with code in major_copy_or_mark_object() */
+
+ /* FIXME: Give destination back to the allocator. */
+ *(void**)destination = NULL;
+
+ vtable_word = *(mword*)obj;
+ g_assert (vtable_word & SGEN_FORWARDED_BIT);
+
+ obj = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
+
+ *obj_slot = obj;
+
+ HEAVY_STAT (++stat_slots_allocated_in_vain);
+ }
+}
+
+#endif
\ No newline at end of file
long long stat_scan_object_called_nursery = 0;
long long stat_scan_object_called_major = 0;
+long long stat_slots_allocated_in_vain;
+
long long stat_nursery_copy_object_failed_from_space = 0;
long long stat_nursery_copy_object_failed_forwarded = 0;
long long stat_nursery_copy_object_failed_pinned = 0;
binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
}
+void
+sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
+{
+ for (;;) {
+ mword vtable_word;
+ gboolean major_pinned = FALSE;
+
+ if (sgen_ptr_in_nursery (obj)) {
+ if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
+ sgen_pin_object (obj, queue);
+ break;
+ }
+ } else {
+ major_collector.pin_major_object (obj, queue);
+ major_pinned = TRUE;
+ }
+
+ vtable_word = *(mword*)obj;
+ /*someone else forwarded it, update the pointer and bail out*/
+ if (vtable_word & SGEN_FORWARDED_BIT) {
+ *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
+ break;
+ }
+
+ /*someone pinned it, nothing to do.*/
+ if (vtable_word & SGEN_PINNED_BIT || major_pinned)
+ break;
+ }
+}
+
/* Sort the addresses in array in increasing order.
* Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
*/
mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
+ mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
+
mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
current_collection_generation = GENERATION_NURSERY;
if (sgen_collection_is_parallel ())
- current_object_ops = major_collector.par_minor_ops;
+ current_object_ops = sgen_minor_collector.parallel_ops;
else
- current_object_ops = major_collector.minor_ops;
+ current_object_ops = sgen_minor_collector.serial_ops;
reset_pinned_from_failed_allocation ();
char* (*alloc_for_promotion) (char *obj, size_t objsize, gboolean has_references);
char* (*par_alloc_for_promotion) (char *obj, size_t objsize, gboolean has_references);
+ SgenObjectOperations serial_ops;
+ SgenObjectOperations parallel_ops;
+
void (*prepare_to_space) (char *to_space_bitmap, int space_bitmap_size);
void (*clear_fragments) (void);
SgenFragment* (*build_fragments_get_exclude_head) (void);
void* (*alloc_degraded) (MonoVTable *vtable, size_t size);
SgenObjectOperations major_ops;
- SgenObjectOperations minor_ops;
- SgenObjectOperations par_minor_ops;
void* (*alloc_object) (int size, gboolean has_references);
void* (*par_alloc_object) (int size, gboolean has_references);
void (*free_non_pinned_object) (char *obj, size_t size);
void (*find_pin_queue_start_ends) (SgenGrayQueue *queue);
void (*pin_objects) (SgenGrayQueue *queue);
+ void (*pin_major_object) (char *obj, SgenGrayQueue *queue);
void (*scan_card_table) (SgenGrayQueue *queue);
void (*iterate_live_block_ranges) (sgen_cardtable_block_callback callback);
void (*init_to_space) (void);
gboolean sgen_try_alloc_space (mword size, int space) MONO_INTERNAL;
void sgen_release_space (mword size, int space) MONO_INTERNAL;
void sgen_pin_object (void *object, SgenGrayQueue *queue) MONO_INTERNAL;
+void sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue) MONO_INTERNAL;
void sgen_collect_major_no_lock (const char *reason) MONO_INTERNAL;
void sgen_collect_nursery_no_lock (size_t requested_size) MONO_INTERNAL;
void sgen_minor_collect_or_expand_inner (size_t size) MONO_INTERNAL;
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
-extern long long stat_copy_object_called_nursery;
-extern long long stat_objects_copied_nursery;
-extern long long stat_nursery_copy_object_failed_from_space;
-extern long long stat_nursery_copy_object_failed_forwarded;
-extern long long stat_nursery_copy_object_failed_pinned;
+#define collector_pin_object(obj, queue) do { \
+ if (sgen_ptr_in_nursery (obj)) { \
+ sgen_pin_object (obj, queue); \
+ } else { \
+ g_assert (objsize <= SGEN_MAX_SMALL_OBJ_SIZE); \
+ pin_major_object (obj, queue); \
+ } \
+} while (0)
-/*
- * This function can be used even if the vtable of obj is not valid
- * anymore, which is the case in the parallel collector.
- */
-static void
-par_copy_object_no_checks (char *destination, MonoVTable *vt, void *obj, mword objsize, SgenGrayQueue *queue)
-{
- static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
-
- DEBUG (9, g_assert (vt->klass->inited));
- DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %lu)\n", destination, ((MonoObject*)obj)->vtable->klass->name, (unsigned long)objsize));
- binary_protocol_copy (obj, destination, vt, objsize);
-
- if (objsize <= sizeof (gpointer) * 8) {
- mword *dest = (mword*)destination;
- goto *copy_labels [objsize / sizeof (gpointer)];
- LAB_8:
- (dest) [7] = ((mword*)obj) [7];
- LAB_7:
- (dest) [6] = ((mword*)obj) [6];
- LAB_6:
- (dest) [5] = ((mword*)obj) [5];
- LAB_5:
- (dest) [4] = ((mword*)obj) [4];
- LAB_4:
- (dest) [3] = ((mword*)obj) [3];
- LAB_3:
- (dest) [2] = ((mword*)obj) [2];
- LAB_2:
- (dest) [1] = ((mword*)obj) [1];
- LAB_1:
- ;
- LAB_0:
- ;
- } else {
- /*can't trust memcpy doing word copies */
- mono_gc_memmove (destination + sizeof (mword), (char*)obj + sizeof (mword), objsize - sizeof (mword));
- }
- /* adjust array->bounds */
- DEBUG (9, g_assert (vt->gc_descr));
- if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
- MonoArray *array = (MonoArray*)destination;
- array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
- DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %lu, rank: %d, length: %lu\n", array, (unsigned long)objsize, vt->rank, (unsigned long)mono_array_length (array)));
- }
- if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES))
- sgen_register_moved_object (obj, destination);
- obj = destination;
- if (queue) {
- DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, sgen_safe_name (obj)));
- GRAY_OBJECT_ENQUEUE (queue, obj);
- }
-}
-
-static void*
-copy_object_no_checks (void *obj, SgenGrayQueue *queue)
-{
- MonoVTable *vt = ((MonoObject*)obj)->vtable;
- gboolean has_references = SGEN_VTABLE_HAS_REFERENCES (vt);
- mword objsize = SGEN_ALIGN_UP (sgen_par_object_get_size (vt, (MonoObject*)obj));
- char *destination = sgen_minor_collector.alloc_for_promotion (obj, objsize, has_references);
-
- if (G_UNLIKELY (!destination)) {
- if (sgen_ptr_in_nursery (obj)) {
- sgen_pin_object (obj, queue);
- } else {
- g_assert (objsize <= SGEN_MAX_SMALL_OBJ_SIZE);
- pin_major_object (obj, queue);
- }
- sgen_set_pinned_from_failed_allocation (objsize);
- return obj;
- }
-
- *(MonoVTable**)destination = vt;
- par_copy_object_no_checks (destination, vt, obj, objsize, has_references ? queue : NULL);
-
- /* set the forwarding pointer */
- SGEN_FORWARD_OBJECT (obj, destination);
-
- return destination;
-}
-
-/*
- * This is how the copying happens from the nursery to the old generation.
- * We assume that at this time all the pinned objects have been identified and
- * marked as such.
- * We run scan_object() for each pinned object so that each referenced
- * objects if possible are copied. The new gray objects created can have
- * scan_object() run on them right away, too.
- * Then we run copy_object() for the precisely tracked roots. At this point
- * all the roots are either gray or black. We run scan_object() on the gray
- * objects until no more gray objects are created.
- * At the end of the process we walk again the pinned list and we unmark
- * the pinned flag. As we go we also create the list of free space for use
- * in the next allocation runs.
- *
- * We need to remember objects from the old generation that point to the new one
- * (or just addresses?).
- *
- * copy_object could be made into a macro once debugged (use inline for now).
- */
-
-static void
-nopar_copy_object (void **obj_slot, SgenGrayQueue *queue)
-{
- char *forwarded;
- char *obj = *obj_slot;
-
- DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
-
- HEAVY_STAT (++stat_copy_object_called_nursery);
-
- if (!sgen_ptr_in_nursery (obj)) {
- HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
- return;
- }
-
- DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
-
- /*
- * Before we can copy the object we must make sure that we are
- * allowed to, i.e. that the object not pinned, not already
- * forwarded or belongs to the nursery To Space.
- */
-
- if ((forwarded = SGEN_OBJECT_IS_FORWARDED (obj))) {
- DEBUG (9, g_assert ((*(MonoVTable**)SGEN_LOAD_VTABLE(obj))->gc_descr));
- DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
- HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
- *obj_slot = forwarded;
- return;
- }
- if (SGEN_OBJECT_IS_PINNED (obj)) {
- DEBUG (9, g_assert (((MonoVTable*)SGEN_LOAD_VTABLE(obj))->gc_descr));
- DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
- HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
- return;
- }
-
- if (sgen_nursery_is_to_space (obj)) {
- DEBUG (9, g_assert (((MonoVTable*)SGEN_LOAD_VTABLE(obj))->gc_descr));
- DEBUG (9, fprintf (gc_debug_file, " (tospace, no change)\n"));
- HEAVY_STAT (++stat_nursery_copy_object_failed_to_space);
- return;
- }
-
- HEAVY_STAT (++stat_objects_copied_nursery);
-
- *obj_slot = copy_object_no_checks (obj, queue);
-}
-
-#ifdef SGEN_PARALLEL_MARK
-
-static void
-copy_object (void **obj_slot, SgenGrayQueue *queue)
-{
- char *obj = *obj_slot;
- mword vtable_word, objsize;
- MonoVTable *vt;
- void *destination;
- gboolean has_references;
-
- DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
-
- HEAVY_STAT (++stat_copy_object_called_nursery);
-
- if (!sgen_ptr_in_nursery (obj)) {
- HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
- return;
- }
-
- vtable_word = *(mword*)obj;
- vt = (MonoVTable*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
-
- /*
- * Before we can copy the object we must make sure that we are
- * allowed to, i.e. that the object not pinned, not already
- * forwarded and not in the nursery To Space.
- */
-
- if (vtable_word & SGEN_FORWARDED_BIT) {
- HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
- *obj_slot = vt;
- return;
- }
- if (vtable_word & SGEN_PINNED_BIT) {
- HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
- return;
- }
-
- if (sgen_nursery_is_to_space (obj)) {
- HEAVY_STAT (++stat_nursery_copy_object_failed_to_space);
- return;
- }
-
- HEAVY_STAT (++stat_objects_copied_nursery);
-
- objsize = SGEN_ALIGN_UP (sgen_par_object_get_size (vt, (MonoObject*)obj));
- has_references = SGEN_VTABLE_HAS_REFERENCES (vt);
-
- destination = sgen_minor_collector.par_alloc_for_promotion (obj, objsize, has_references);
-
- if (G_UNLIKELY (!destination)) {
- pin_or_update_par (obj_slot, obj, vt, queue);
- return;
- }
-
- *(MonoVTable**)destination = vt;
-
- if (SGEN_CAS_PTR ((void*)obj, (void*)((mword)destination | SGEN_FORWARDED_BIT), vt) == vt) {
- par_copy_object_no_checks (destination, vt, obj, objsize, has_references ? queue : NULL);
- obj = destination;
- *obj_slot = obj;
- } else {
- /* FIXME: unify with code in major_copy_or_mark_object() */
-
- /* FIXME: Give destination back to the allocator. */
- *(void**)destination = NULL;
-
- vtable_word = *(mword*)obj;
- g_assert (vtable_word & SGEN_FORWARDED_BIT);
-
- obj = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
-
- *obj_slot = obj;
+#define collector_serial_alloc_for_promotion sgen_minor_collector.alloc_for_promotion
+#define collector_parallel_alloc_for_promotion sgen_minor_collector.par_alloc_for_promotion
- ++stat_slots_allocated_in_vain;
- }
-}
-#else
-static void
-copy_object (void **obj_slot, SgenGrayQueue *queue)
-{
- nopar_copy_object (obj_slot, queue);
-}
-#endif
-#define FILL_COLLECTOR_COPY_OBJECT(collector) do { \
- (collector)->minor_ops.copy_or_mark_object = nopar_copy_object; \
- (collector)->par_minor_ops.copy_or_mark_object = copy_object; \
- } while (0)
+#include "sgen-copy-object.h"
\ No newline at end of file
return p;
}
-#define pin_major_object sgen_pin_object
+static inline void
+pin_major_object (char *obj, SgenGrayQueue *queue)
+{
+ sgen_pin_object (obj, queue);
+}
#include "sgen-major-copy-object.h"
collector->free_non_pinned_object = major_free_non_pinned_object;
collector->find_pin_queue_start_ends = major_find_pin_queue_start_ends;
collector->pin_objects = major_pin_objects;
+ collector->pin_major_object = pin_major_object;
collector->init_to_space = major_init_to_space;
collector->sweep = major_sweep;
collector->check_scan_starts = major_check_scan_starts;
collector->print_gc_param_usage = NULL;
collector->major_ops.copy_or_mark_object = major_copy_or_mark_object;
- FILL_COLLECTOR_COPY_OBJECT (collector);
- FILL_COLLECTOR_SCAN_OBJECT (collector);
+ collector->major_ops.scan_object = major_scan_object;
}
#endif
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
-extern long long stat_scan_object_called_nursery;
extern long long stat_scan_object_called_major;
-#undef HANDLE_PTR
-#define HANDLE_PTR(ptr,obj) do { \
- void *__old = *(ptr); \
- void *__copy; \
- if (__old) { \
- copy_object ((ptr), queue); \
- __copy = *(ptr); \
- DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
- if (G_UNLIKELY (sgen_ptr_in_nursery (__copy) && !sgen_ptr_in_nursery ((ptr)))) \
- sgen_add_to_global_remset ((ptr)); \
- } \
- } while (0)
-
-/*
- * Scan the object pointed to by @start for references to
- * other objects between @from_start and @from_end and copy
- * them to the gray_objects area.
- */
-static void
-minor_scan_object (char *start, SgenGrayQueue *queue)
-{
-#include "sgen-scan-object.h"
-
- HEAVY_STAT (++stat_scan_object_called_nursery);
-}
-
-/*
- * scan_vtype:
- *
- * Scan the valuetype pointed to by START, described by DESC for references to
- * other objects between @from_start and @from_end and copy them to the gray_objects area.
- * Returns a pointer to the end of the object.
- */
-static void
-minor_scan_vtype (char *start, mword desc, SgenGrayQueue *queue)
-{
- /* The descriptors include info about the MonoObject header as well */
- start -= sizeof (MonoObject);
-
-#define SCAN_OBJECT_NOVTABLE
-#include "sgen-scan-object.h"
-}
-
-#undef HANDLE_PTR
-#define HANDLE_PTR(ptr,obj) do { \
- void *__old = *(ptr); \
- void *__copy; \
- if (__old) { \
- nopar_copy_object ((ptr), queue); \
- __copy = *(ptr); \
- DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
- if (G_UNLIKELY (sgen_ptr_in_nursery (__copy) && !sgen_ptr_in_nursery ((ptr)))) \
- sgen_add_to_global_remset ((ptr)); \
- } \
- } while (0)
-
-static void
-nopar_minor_scan_object (char *start, SgenGrayQueue *queue)
-{
-#include "sgen-scan-object.h"
-
- HEAVY_STAT (++stat_scan_object_called_nursery);
-}
-
-static void
-nopar_minor_scan_vtype (char *start, mword desc, SgenGrayQueue *queue)
-{
- /* The descriptors include info about the MonoObject header as well */
- start -= sizeof (MonoObject);
-
-#define SCAN_OBJECT_NOVTABLE
-#include "sgen-scan-object.h"
-}
-
#ifdef FIXED_HEAP
#define PREFETCH_DYNAMIC_HEAP(addr)
#else
HEAVY_STAT (++stat_scan_object_called_major);
}
-
-#define FILL_COLLECTOR_SCAN_OBJECT(collector) do { \
- (collector)->major_ops.scan_object = major_scan_object; \
- (collector)->par_minor_ops.scan_object = minor_scan_object; \
- (collector)->minor_ops.scan_object = nopar_minor_scan_object; \
- (collector)->par_minor_ops.scan_vtype = minor_scan_vtype; \
- (collector)->minor_ops.scan_vtype = nopar_minor_scan_vtype; \
- } while (0)
static long long stat_major_blocks_freed = 0;
static long long stat_major_objects_evacuated = 0;
static long long stat_time_wait_for_sweep = 0;
-#ifdef SGEN_PARALLEL_MARK
-static long long stat_slots_allocated_in_vain = 0;
-#endif
static gboolean ms_sweep_in_progress = FALSE;
static MonoNativeThreadId ms_sweep_thread;
} while (0)
static void
-pin_major_object (void *obj, SgenGrayQueue *queue)
+pin_major_object (char *obj, SgenGrayQueue *queue)
{
MSBlockInfo *block = MS_BLOCK_FOR_OBJ (obj);
block->has_pinned = TRUE;
MS_MARK_OBJECT_AND_ENQUEUE (obj, block, queue);
}
-#ifdef SGEN_PARALLEL_MARK
-static void
-pin_or_update_par (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
-{
- for (;;) {
- mword vtable_word;
- gboolean major_pinned = FALSE;
-
- if (sgen_ptr_in_nursery (obj)) {
- if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
- sgen_pin_object (obj, queue);
- break;
- }
- } else {
- pin_major_object (obj, queue);
- major_pinned = TRUE;
- }
-
- vtable_word = *(mword*)obj;
- /*someone else forwarded it, update the pointer and bail out*/
- if (vtable_word & SGEN_FORWARDED_BIT) {
- *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
- break;
- }
-
- /*someone pinned it, nothing to do.*/
- if (vtable_word & SGEN_PINNED_BIT || major_pinned)
- break;
- }
-}
-#endif
-
#include "sgen-major-copy-object.h"
#ifdef SGEN_PARALLEL_MARK
evacuate_block_obj_sizes [size_index] = FALSE;
}
- pin_or_update_par (ptr, obj, vt, queue);
+ sgen_parallel_pin_or_update (ptr, obj, vt, queue);
sgen_set_pinned_from_failed_allocation (objsize);
return;
}
*ptr = obj;
- ++stat_slots_allocated_in_vain;
+ HEAVY_STAT (++stat_slots_allocated_in_vain);
}
} else {
#ifdef FIXED_HEAP
mono_counters_register ("# major objects evacuated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_major_objects_evacuated);
mono_counters_register ("Wait for sweep time", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &stat_time_wait_for_sweep);
#ifdef SGEN_PARALLEL_MARK
- mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
-
#ifndef HAVE_KW_THREAD
mono_native_tls_alloc (&workers_free_block_lists_key, NULL);
#endif
collector->free_non_pinned_object = major_free_non_pinned_object;
collector->find_pin_queue_start_ends = major_find_pin_queue_start_ends;
collector->pin_objects = major_pin_objects;
+ collector->pin_major_object = pin_major_object;
#ifdef SGEN_HAVE_CARDTABLE
collector->scan_card_table = major_scan_card_table;
collector->iterate_live_block_ranges = (void*)(void*) major_iterate_live_block_ranges;
/* FIXME this macro mess */
collector->major_ops.copy_or_mark_object = major_copy_or_mark_object;
- FILL_COLLECTOR_COPY_OBJECT (collector);
- FILL_COLLECTOR_SCAN_OBJECT (collector);
+ collector->major_ops.scan_object = major_scan_object;
#ifdef SGEN_HAVE_CARDTABLE
/*cardtable requires major pages to be 8 cards aligned*/
--- /dev/null
+/*
+ * Copyright 2001-2003 Ximian, Inc
+ * Copyright 2003-2010 Novell, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#define collector_pin_object(obj, queue) sgen_pin_object (obj, queue);
+#define collector_serial_alloc_for_promotion alloc_for_promotion
+#define collector_parallel_alloc_for_promotion par_alloc_for_promotion
+
+#define GENERATE_COPY_FUNCTIONS 1
+
+
+#include "sgen-copy-object.h"
+
+#define FILL_MINOR_COLLECTOR_COPY_OBJECT(collector) do { \
+ (collector)->serial_ops.copy_or_mark_object = serial_copy_object; \
+ (collector)->parallel_ops.copy_or_mark_object = parallel_copy_object; \
+ } while (0)
--- /dev/null
+/*
+ * Copyright 2001-2003 Ximian, Inc
+ * Copyright 2003-2010 Novell, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+extern long long stat_scan_object_called_nursery;
+
+#undef HANDLE_PTR
+#define HANDLE_PTR(ptr,obj) do { \
+ void *__old = *(ptr); \
+ void *__copy; \
+ if (__old) { \
+ parallel_copy_object ((ptr), queue); \
+ __copy = *(ptr); \
+ DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
+ if (G_UNLIKELY (sgen_ptr_in_nursery (__copy) && !sgen_ptr_in_nursery ((ptr)))) \
+ sgen_add_to_global_remset ((ptr)); \
+ } \
+ } while (0)
+
+/*
+ * Scan the object pointed to by @start for references to
+ * other objects between @from_start and @from_end and copy
+ * them to the gray_objects area.
+ */
+static void
+parallel_scan_object (char *start, SgenGrayQueue *queue)
+{
+#include "sgen-scan-object.h"
+
+ HEAVY_STAT (++stat_scan_object_called_nursery);
+}
+
+/*
+ * scan_vtype:
+ *
+ * Scan the valuetype pointed to by START, described by DESC for references to
+ * other objects between @from_start and @from_end and copy them to the gray_objects area.
+ * Returns a pointer to the end of the object.
+ */
+static void
+parallel_scan_vtype (char *start, mword desc, SgenGrayQueue *queue)
+{
+ /* The descriptors include info about the MonoObject header as well */
+ start -= sizeof (MonoObject);
+
+#define SCAN_OBJECT_NOVTABLE
+#include "sgen-scan-object.h"
+}
+
+#undef HANDLE_PTR
+#define HANDLE_PTR(ptr,obj) do { \
+ void *__old = *(ptr); \
+ void *__copy; \
+ if (__old) { \
+ serial_copy_object ((ptr), queue); \
+ __copy = *(ptr); \
+ DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
+ if (G_UNLIKELY (sgen_ptr_in_nursery (__copy) && !sgen_ptr_in_nursery ((ptr)))) \
+ sgen_add_to_global_remset ((ptr)); \
+ } \
+ } while (0)
+
+static void
+serial_scan_object (char *start, SgenGrayQueue *queue)
+{
+#include "sgen-scan-object.h"
+
+ HEAVY_STAT (++stat_scan_object_called_nursery);
+}
+
+static void
+serial_scan_vtype (char *start, mword desc, SgenGrayQueue *queue)
+{
+ /* The descriptors include info about the MonoObject header as well */
+ start -= sizeof (MonoObject);
+
+#define SCAN_OBJECT_NOVTABLE
+#include "sgen-scan-object.h"
+}
+
+#define FILL_MINOR_COLLECTOR_SCAN_OBJECT(collector) do { \
+ (collector)->parallel_ops.scan_object = parallel_scan_object; \
+ (collector)->parallel_ops.scan_vtype = parallel_scan_vtype; \
+ (collector)->serial_ops.scan_object = serial_scan_object; \
+ (collector)->serial_ops.scan_vtype = serial_scan_vtype; \
+ } while (0)
#include "config.h"
#ifdef HAVE_SGEN_GC
+#include "metadata/profiler-private.h"
+
#include "metadata/sgen-gc.h"
+#include "metadata/sgen-protocol.h"
-static char*
+static inline char*
alloc_for_promotion (char *obj, size_t objsize, gboolean has_references)
{
return major_collector.alloc_object (objsize, has_references);
}
-static char*
+static inline char*
par_alloc_for_promotion (char *obj, size_t objsize, gboolean has_references)
{
return major_collector.par_alloc_object (objsize, has_references);
sgen_fragment_allocator_add (allocator, start, end);
}
+
+/******************************************Copy/Scan functins ************************************************/
+
+#include "sgen-minor-copy-object.h"
+#include "sgen-minor-scan-object.h"
+
void
sgen_simple_nursery_init (SgenMinorCollector *collector)
{
collector->build_fragments_release_exclude_head = build_fragments_release_exclude_head;
collector->build_fragments_finish = build_fragments_finish;
collector->init_nursery = init_nursery;
+
+ FILL_MINOR_COLLECTOR_COPY_OBJECT (collector);
+ FILL_MINOR_COLLECTOR_SCAN_OBJECT (collector);
}
#include "config.h"
#ifdef HAVE_SGEN_GC
+#include "metadata/profiler-private.h"
+
#include "metadata/sgen-gc.h"
+#include "metadata/sgen-protocol.h"
/*
The nursery is logically divided into 3 spaces: Allocator space and two Survivor spaces.
-We apply the same promotion policy to all objects, finalizable ones should age longer in the nursery;
-We apply the same promotion policy to all stages of a collection, maybe we should promote more aggressively
objects from non-stack roots, specially those found in the remembered set;
--Make the new behavior runtime selectable;
--Make the new behavior have a low overhead when disabled;
--Make all new exported functions inlineable in other modules;
--Create specialized copy & scan functions for nursery collections;
--Decide if this is the right place for this code;
-Fix our major collection trigger to happen before we do a minor GC and collect the nursery only once.
+-Make the serial fragment allocator fast path inlineable
*/
/*FIXME Move this to a separate header. */
}
/******************************************Minor Collector API ************************************************/
-static char*
+static inline char*
alloc_for_promotion (char *obj, size_t objsize, gboolean has_references)
{
char *p;
return p;
}
-static char*
+static inline char*
par_alloc_for_promotion (char *obj, size_t objsize, gboolean has_references)
{
char *p;
promotion_barrier = middle;
}
+/******************************************Copy/Scan functins ************************************************/
+
+#include "sgen-minor-copy-object.h"
+#include "sgen-minor-scan-object.h"
+
+
void
sgen_split_nursery_init (SgenMinorCollector *collector)
{
collector->build_fragments_release_exclude_head = build_fragments_release_exclude_head;
collector->build_fragments_finish = build_fragments_finish;
collector->init_nursery = init_nursery;
+
+ FILL_MINOR_COLLECTOR_COPY_OBJECT (collector);
+ FILL_MINOR_COLLECTOR_SCAN_OBJECT (collector);
}
projects / mono.git / commitdiff