[sgen] Only do fence when concurrent collection is running.

The store/store fence incurs quite a high cost, so we only want it to
run when absolutely necessary.  The most efficient way is to
separately compile nursery object operations for when the concurrent
collector is running, so there has to be no dynamic check.

In the future we might want to try batching updates of references to
moved objects so we have to do only one fence per batch.

diff --git a/mono/sgen/sgen-gc.c b/mono/sgen/sgen-gc.c
index 1223ca01ba54c8461715c8f732bdc5bde0272fee..d4240da9bfecafd5c04f3068afb840d46052f5e6 100644 (file)
--- a/mono/sgen/sgen-gc.c
+++ b/mono/sgen/sgen-gc.c
@@ -1472,7 +1472,7 @@ collect_nursery (const char *reason, gboolean is_overflow, SgenGrayQueue *unpin_
        mword fragment_total;
        ScanJob *sj;
        SgenGrayQueue gc_thread_gray_queue;
-       SgenObjectOperations *object_ops = &sgen_minor_collector.serial_ops;
+       SgenObjectOperations *object_ops;
        ScanCopyContext ctx;
        TV_DECLARE (atv);
        TV_DECLARE (btv);
@@ -1485,6 +1485,11 @@ collect_nursery (const char *reason, gboolean is_overflow, SgenGrayQueue *unpin_
 
        binary_protocol_collection_begin (gc_stats.minor_gc_count, GENERATION_NURSERY);
 
+       if (sgen_concurrent_collection_in_progress ())
+               object_ops = &sgen_minor_collector.serial_ops_with_concurrent_major;
+       else
+               object_ops = &sgen_minor_collector.serial_ops;
+
        if (do_verify_nursery || do_dump_nursery_content)
                sgen_debug_verify_nursery (do_dump_nursery_content);
 

diff --git a/mono/sgen/sgen-gc.h b/mono/sgen/sgen-gc.h
index 94073fb505e2bc54eadf785a5c792db2d7fec7ab..0674c7fea1b80d94527e0f361d70f741ad2ad3e5 100644 (file)
--- a/mono/sgen/sgen-gc.h
+++ b/mono/sgen/sgen-gc.h
@@ -557,6 +557,7 @@ typedef struct {
        GCObject* (*alloc_for_promotion) (GCVTable vtable, GCObject *obj, size_t objsize, gboolean has_references);
 
        SgenObjectOperations serial_ops;
+       SgenObjectOperations serial_ops_with_concurrent_major;
 
        void (*prepare_to_space) (char *to_space_bitmap, size_t space_bitmap_size);
        void (*clear_fragments) (void);

diff --git a/mono/sgen/sgen-minor-copy-object.h b/mono/sgen/sgen-minor-copy-object.h
index e6d6a40b467b15b08948ee8bcb4454308c036e4c..da392c4292847badfe7fcc5a9b01e9c2cb082851 100644 (file)
--- a/mono/sgen/sgen-minor-copy-object.h
+++ b/mono/sgen/sgen-minor-copy-object.h
@@ -8,12 +8,35 @@
  * Licensed under the MIT license. See LICENSE file in the project root for full license information.
  */
 
-#define collector_pin_object(obj, queue) sgen_pin_object (obj, queue);
-#define COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION alloc_for_promotion
+#undef SERIAL_COPY_OBJECT
+#undef SERIAL_COPY_OBJECT_FROM_OBJ
 
-extern guint64 stat_nursery_copy_object_failed_to_space; /* from sgen-gc.c */
+#if defined(SGEN_SIMPLE_NURSERY)
+
+#ifdef SGEN_CONCURRENT_MAJOR
+#define SERIAL_COPY_OBJECT simple_nursery_serial_with_concurrent_major_copy_object
+#define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_with_concurrent_major_copy_object_from_obj
+#else
+#define SERIAL_COPY_OBJECT simple_nursery_serial_copy_object
+#define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_copy_object_from_obj
+#endif
+
+#elif defined (SGEN_SPLIT_NURSERY)
+
+#ifdef SGEN_CONCURRENT_MAJOR
+#define SERIAL_COPY_OBJECT split_nursery_serial_with_concurrent_major_copy_object
+#define SERIAL_COPY_OBJECT_FROM_OBJ split_nursery_serial_with_concurrent_major_copy_object_from_obj
+#else
+#define SERIAL_COPY_OBJECT split_nursery_serial_copy_object
+#define SERIAL_COPY_OBJECT_FROM_OBJ split_nursery_serial_copy_object_from_obj
+#endif
 
-#include "sgen-copy-object.h"
+#else
+#error "No nursery configuration specified"
+#endif
+
+
+extern guint64 stat_nursery_copy_object_failed_to_space; /* from sgen-gc.c */
 
 /*
  * This is how the copying happens from the nursery to the old generation.
@@ -121,8 +144,10 @@ SERIAL_COPY_OBJECT_FROM_OBJ (GCObject **obj_slot, SgenGrayQueue *queue)
                SGEN_ASSERT (9, sgen_obj_get_descriptor (forwarded),  "forwarded object %p has no gc descriptor", forwarded);
                SGEN_LOG (9, " (already forwarded to %p)", forwarded);
                HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
+#ifdef SGEN_CONCURRENT_MAJOR
                /* See comment on STORE_STORE_FENCE below. */
                STORE_STORE_FENCE;
+#endif
                SGEN_UPDATE_REFERENCE (obj_slot, forwarded);
 #ifndef SGEN_SIMPLE_NURSERY
                if (G_UNLIKELY (sgen_ptr_in_nursery (forwarded) && !sgen_ptr_in_nursery (obj_slot) && !SGEN_OBJECT_IS_CEMENTED (forwarded)))
@@ -189,17 +214,16 @@ SERIAL_COPY_OBJECT_FROM_OBJ (GCObject **obj_slot, SgenGrayQueue *queue)
        HEAVY_STAT (++stat_objects_copied_nursery);
 
        copy = copy_object_no_checks (obj, queue);
+#ifdef SGEN_CONCURRENT_MAJOR
        /*
         * If an object is evacuated to the major heap and a reference to it, from the major
         * heap, updated, the concurrent major collector might follow that reference and
         * scan the new major object.  To make sure the object contents are seen by the
         * major collector we need this write barrier, so that the reference is seen after
         * the object.
-        *
-        * FIXME: This is only required if there's a concurrent major collection running,
-        * but we can't conditionally compile on that at the moment.
         */
        STORE_STORE_FENCE;
+#endif
        SGEN_UPDATE_REFERENCE (obj_slot, copy);
 #ifndef SGEN_SIMPLE_NURSERY
        if (G_UNLIKELY (sgen_ptr_in_nursery (copy) && !sgen_ptr_in_nursery (obj_slot) && !SGEN_OBJECT_IS_CEMENTED (copy)))
@@ -212,7 +236,3 @@ SERIAL_COPY_OBJECT_FROM_OBJ (GCObject **obj_slot, SgenGrayQueue *queue)
        }
 #endif
 }
-
-#define FILL_MINOR_COLLECTOR_COPY_OBJECT(collector)    do {                    \
-               (collector)->serial_ops.copy_or_mark_object = SERIAL_COPY_OBJECT;                       \
-       } while (0)

diff --git a/mono/sgen/sgen-minor-scan-object.h b/mono/sgen/sgen-minor-scan-object.h
index 4b62a739f27183622a0dbc4441e1ea2b17536b9c..cc3e7a4539e794037d54c6055196b2d9df815fef 100644 (file)
--- a/mono/sgen/sgen-minor-scan-object.h
+++ b/mono/sgen/sgen-minor-scan-object.h
@@ -10,16 +10,36 @@
 
 extern guint64 stat_scan_object_called_nursery;
 
+#undef SERIAL_SCAN_OBJECT
+#undef SERIAL_SCAN_VTYPE
+#undef SERIAL_SCAN_PTR_FIELD
+
 #if defined(SGEN_SIMPLE_NURSERY)
+
+#ifdef SGEN_CONCURRENT_MAJOR
+#define SERIAL_SCAN_OBJECT simple_nursery_serial_with_concurrent_major_scan_object
+#define SERIAL_SCAN_VTYPE simple_nursery_serial_with_concurrent_major_scan_vtype
+#define SERIAL_SCAN_PTR_FIELD simple_nursery_serial_with_concurrent_major_scan_ptr_field
+#else
 #define SERIAL_SCAN_OBJECT simple_nursery_serial_scan_object
 #define SERIAL_SCAN_VTYPE simple_nursery_serial_scan_vtype
+#define SERIAL_SCAN_PTR_FIELD simple_nursery_serial_scan_ptr_field
+#endif
 
 #elif defined (SGEN_SPLIT_NURSERY)
+
+#ifdef SGEN_CONCURRENT_MAJOR
+#define SERIAL_SCAN_OBJECT split_nursery_serial_with_concurrent_major_scan_object
+#define SERIAL_SCAN_VTYPE split_nursery_serial_with_concurrent_major_scan_vtype
+#define SERIAL_SCAN_PTR_FIELD split_nursery_serial_with_concurrent_major_scan_ptr_field
+#else
 #define SERIAL_SCAN_OBJECT split_nursery_serial_scan_object
 #define SERIAL_SCAN_VTYPE split_nursery_serial_scan_vtype
+#define SERIAL_SCAN_PTR_FIELD split_nursery_serial_scan_ptr_field
+#endif
 
 #else
-#error "Please define GC_CONF_NAME"
+#error "No nursery configuration specified"
 #endif
 
 #undef HANDLE_PTR
@@ -75,8 +95,8 @@ SERIAL_SCAN_PTR_FIELD (GCObject *full_object, GCObject **ptr, SgenGrayQueue *que
        HANDLE_PTR (ptr, NULL);
 }
 
-#define FILL_MINOR_COLLECTOR_SCAN_OBJECT(collector)    do {                    \
-               (collector)->serial_ops.scan_object = SERIAL_SCAN_OBJECT;       \
-               (collector)->serial_ops.scan_vtype = SERIAL_SCAN_VTYPE; \
-               (collector)->serial_ops.scan_ptr_field = SERIAL_SCAN_PTR_FIELD; \
+#define FILL_MINOR_COLLECTOR_SCAN_OBJECT(ops)  do {                    \
+               (ops)->scan_object = SERIAL_SCAN_OBJECT;                        \
+               (ops)->scan_vtype = SERIAL_SCAN_VTYPE;                  \
+               (ops)->scan_ptr_field = SERIAL_SCAN_PTR_FIELD;          \
        } while (0)

diff --git a/mono/sgen/sgen-simple-nursery.c b/mono/sgen/sgen-simple-nursery.c
index 1004957d2611c087060ec0e5c8a400d339eaa1ac..bc8aee7044c8a12baaa684e9411893dedf9f567a 100644 (file)
--- a/mono/sgen/sgen-simple-nursery.c
+++ b/mono/sgen/sgen-simple-nursery.c
@@ -62,14 +62,35 @@ init_nursery (SgenFragmentAllocator *allocator, char *start, char *end)
 
 /******************************************Copy/Scan functins ************************************************/
 
+#define collector_pin_object(obj, queue) sgen_pin_object (obj, queue);
+#define COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION alloc_for_promotion
+
+#include "sgen-copy-object.h"
+
 #define SGEN_SIMPLE_NURSERY
 
-#define SERIAL_COPY_OBJECT simple_nursery_serial_copy_object
-#define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_copy_object_from_obj
+#include "sgen-minor-copy-object.h"
+#include "sgen-minor-scan-object.h"
+
+static void
+fill_serial_ops (SgenObjectOperations *ops)
+{
+       ops->copy_or_mark_object = SERIAL_COPY_OBJECT;
+       FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
+}
+
+#define SGEN_CONCURRENT_MAJOR
 
 #include "sgen-minor-copy-object.h"
 #include "sgen-minor-scan-object.h"
 
+static void
+fill_serial_with_concurrent_major_ops (SgenObjectOperations *ops)
+{
+       ops->copy_or_mark_object = SERIAL_COPY_OBJECT;
+       FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
+}
+
 void
 sgen_simple_nursery_init (SgenMinorCollector *collector)
 {
@@ -84,8 +105,8 @@ sgen_simple_nursery_init (SgenMinorCollector *collector)
        collector->build_fragments_finish = build_fragments_finish;
        collector->init_nursery = init_nursery;
 
-       FILL_MINOR_COLLECTOR_COPY_OBJECT (collector);
-       FILL_MINOR_COLLECTOR_SCAN_OBJECT (collector);
+       fill_serial_ops (&collector->serial_ops);
+       fill_serial_with_concurrent_major_ops (&collector->serial_ops_with_concurrent_major);
 }
 
 

diff --git a/mono/sgen/sgen-split-nursery.c b/mono/sgen/sgen-split-nursery.c
index 3b7ae668a203d537bd2372d15d62e7630c5db7e6..a4deb5a3c6a702d2e65b37c1530102f8767776dc 100644 (file)
--- a/mono/sgen/sgen-split-nursery.c
+++ b/mono/sgen/sgen-split-nursery.c
@@ -418,14 +418,35 @@ print_gc_param_usage (void)
 
 /******************************************Copy/Scan functins ************************************************/
 
+#define collector_pin_object(obj, queue) sgen_pin_object (obj, queue);
+#define COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION alloc_for_promotion
+
+#include "sgen-copy-object.h"
+
 #define SGEN_SPLIT_NURSERY
 
-#define SERIAL_COPY_OBJECT split_nursery_serial_copy_object
-#define SERIAL_COPY_OBJECT_FROM_OBJ split_nursery_serial_copy_object_from_obj
+#include "sgen-minor-copy-object.h"
+#include "sgen-minor-scan-object.h"
+
+static void
+fill_serial_ops (SgenObjectOperations *ops)
+{
+       ops->copy_or_mark_object = SERIAL_COPY_OBJECT;
+       FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
+}
+
+#define SGEN_CONCURRENT_MAJOR
 
 #include "sgen-minor-copy-object.h"
 #include "sgen-minor-scan-object.h"
 
+static void
+fill_serial_with_concurrent_major_ops (SgenObjectOperations *ops)
+{
+       ops->copy_or_mark_object = SERIAL_COPY_OBJECT;
+       FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
+}
+
 void
 sgen_split_nursery_init (SgenMinorCollector *collector)
 {
@@ -442,8 +463,8 @@ sgen_split_nursery_init (SgenMinorCollector *collector)
        collector->handle_gc_param = handle_gc_param;
        collector->print_gc_param_usage = print_gc_param_usage;
 
-       FILL_MINOR_COLLECTOR_COPY_OBJECT (collector);
-       FILL_MINOR_COLLECTOR_SCAN_OBJECT (collector);
+       fill_serial_ops (&collector->serial_ops);
+       fill_serial_with_concurrent_major_ops (&collector->serial_ops_with_concurrent_major);
 }