Merge pull request #2800 from BrzVlad/feature-lazy-sweep

[sgen] Enable lazy sweep by default

diff --git a/mono/metadata/sgen-client-mono.h b/mono/metadata/sgen-client-mono.h
index 54630c6ee033853a8ac63065b70eb7cfa1f3a100..e9f9b1a81a7a5ddeaad89a7ae0ddac596f328fcc 100644 (file)
--- a/mono/metadata/sgen-client-mono.h
+++ b/mono/metadata/sgen-client-mono.h
@@ -689,6 +689,11 @@ sgen_client_binary_protocol_evacuating_blocks (size_t block_size)
 {
 }
 
+static void G_GNUC_UNUSED
+sgen_client_binary_protocol_concurrent_sweep_end (long long timestamp)
+{
+}
+
 int sgen_thread_handshake (BOOL suspend);
 gboolean sgen_suspend_thread (SgenThreadInfo *info);
 gboolean sgen_resume_thread (SgenThreadInfo *info);

diff --git a/mono/sgen/sgen-marksweep.c b/mono/sgen/sgen-marksweep.c
index 38e127b8cd541fe4b304adf4f3e91d19135265db..d2e3ccdd0093dcc9e68432ef1e8f20b9198651a2 100644 (file)
--- a/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@ -162,7 +162,7 @@ static int fast_block_obj_size_indexes [MS_NUM_FAST_BLOCK_OBJ_SIZE_INDEXES];
 static gboolean *evacuate_block_obj_sizes;
 static float evacuation_threshold = 0.666f;
 
-static gboolean lazy_sweep = FALSE;
+static gboolean lazy_sweep = TRUE;
 
 enum {
        SWEEP_STATE_SWEPT,
@@ -807,6 +807,7 @@ set_sweep_state (int new_, int expected)
 static gboolean ensure_block_is_checked_for_sweeping (guint32 block_index, gboolean wait, gboolean *have_checked);
 
 static SgenThreadPoolJob * volatile sweep_job;
+static SgenThreadPoolJob * volatile sweep_blocks_job;
 
 static void
 major_finish_sweep_checking (void)
@@ -1577,6 +1578,20 @@ ensure_block_is_checked_for_sweeping (guint32 block_index, gboolean wait, gboole
        return !!tagged_block;
 }
 
+static void
+sweep_blocks_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
+{
+       volatile gpointer *slot;
+
+       SGEN_ARRAY_LIST_FOREACH_SLOT (&allocated_blocks, slot) {
+               sweep_block (BLOCK_UNTAG (*slot));
+       } SGEN_ARRAY_LIST_END_FOREACH_SLOT;
+
+       mono_memory_write_barrier ();
+
+       sweep_blocks_job = NULL;
+}
+
 static void
 sweep_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
 {
@@ -1618,6 +1633,16 @@ sweep_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
 
        sgen_array_list_remove_nulls (&allocated_blocks);
 
+       /*
+        * Concurrently sweep all the blocks to reduce workload during minor
+        * pauses where we need certain blocks to be swept. At the start of
+        * the next major we need all blocks to be swept anyway.
+        */
+       if (concurrent_sweep && lazy_sweep) {
+               sweep_blocks_job = sgen_thread_pool_job_alloc ("sweep_blocks", sweep_blocks_job_func, sizeof (SgenThreadPoolJob));
+               sgen_thread_pool_job_enqueue (sweep_blocks_job);
+       }
+
        sweep_finish ();
 
        sweep_job = NULL;
@@ -1644,6 +1669,8 @@ sweep_finish (void)
        sgen_memgov_major_post_sweep ();
 
        set_sweep_state (SWEEP_STATE_SWEPT, SWEEP_STATE_COMPACTING);
+       if (concurrent_sweep)
+               binary_protocol_concurrent_sweep_end (sgen_timestamp ());
 }
 
 static void
@@ -1863,12 +1890,22 @@ major_start_major_collection (void)
                sgen_evacuation_freelist_blocks (&free_block_lists [MS_BLOCK_FLAG_REFS][i], i);
        }
 
-       if (lazy_sweep)
-               binary_protocol_sweep_begin (GENERATION_OLD, TRUE);
+       if (lazy_sweep && concurrent_sweep) {
+               /*
+                * sweep_blocks_job is created before sweep_finish, which we wait for above
+                * (major_finish_sweep_checking). After the end of sweep, if we don't have
+                * sweep_blocks_job set, it means that it has already been run.
+                */
+               SgenThreadPoolJob *job = sweep_blocks_job;
+               if (job)
+                       sgen_thread_pool_job_wait (job);
+       }
 
+       if (lazy_sweep && !concurrent_sweep)
+               binary_protocol_sweep_begin (GENERATION_OLD, TRUE);
        /* Sweep all unswept blocks and set them to MARKING */
        FOREACH_BLOCK_NO_LOCK (block) {
-               if (lazy_sweep)
+               if (lazy_sweep && !concurrent_sweep)
                        sweep_block (block);
                SGEN_ASSERT (0, block->state == BLOCK_STATE_SWEPT, "All blocks must be swept when we're pinning.");
                set_block_state (block, BLOCK_STATE_MARKING, BLOCK_STATE_SWEPT);
@@ -1880,8 +1917,7 @@ major_start_major_collection (void)
                if (evacuate_block_obj_sizes [block->obj_size_index] && !block->free_list)
                        block->is_to_space = TRUE;
        } END_FOREACH_BLOCK_NO_LOCK;
-
-       if (lazy_sweep)
+       if (lazy_sweep && !concurrent_sweep)
                binary_protocol_sweep_end (GENERATION_OLD, TRUE);
 
        set_sweep_state (SWEEP_STATE_NEED_SWEEPING, SWEEP_STATE_SWEPT);

diff --git a/mono/sgen/sgen-protocol-def.h b/mono/sgen/sgen-protocol-def.h
index 8a190ef6f494df314fa590d74a0022af78f28294..3ab36e3bf660f380b9b21cf554f0e659899eb520 100644 (file)
--- a/mono/sgen/sgen-protocol-def.h
+++ b/mono/sgen/sgen-protocol-def.h
@@ -435,6 +435,13 @@ MATCH_INDEX (BINARY_PROTOCOL_MATCH)
 IS_VTABLE_MATCH (FALSE)
 END_PROTOCOL_ENTRY
 
+BEGIN_PROTOCOL_ENTRY1 (binary_protocol_concurrent_sweep_end, TYPE_LONGLONG, timestamp)
+DEFAULT_PRINT ()
+IS_ALWAYS_MATCH (TRUE)
+MATCH_INDEX (BINARY_PROTOCOL_MATCH)
+IS_VTABLE_MATCH (FALSE)
+END_PROTOCOL_ENTRY
+
 #undef BEGIN_PROTOCOL_ENTRY0
 #undef BEGIN_PROTOCOL_ENTRY1
 #undef BEGIN_PROTOCOL_ENTRY2