[sgen] Concurrent sweep.

diff --git a/mono/metadata/sgen-gc.c b/mono/metadata/sgen-gc.c
index 3057429010aeb51fedcf45ebd0552ed4a2e8067f..a899aa2b2388d5b8eb52719cdba591c1f281072c 100644 (file)
--- a/mono/metadata/sgen-gc.c
+++ b/mono/metadata/sgen-gc.c
@@ -341,10 +341,10 @@ mono_gc_flush_info (void)
  */
 #define USER_CONFIG 1
 
-#define TV_DECLARE(name) gint64 name
-#define TV_GETTIME(tv) tv = mono_100ns_ticks ()
-#define TV_ELAPSED(start,end) (int)((end-start) / 10)
-#define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
+#define TV_DECLARE SGEN_TV_DECLARE
+#define TV_GETTIME SGEN_TV_GETTIME
+#define TV_ELAPSED SGEN_TV_ELAPSED
+#define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
 
 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
 
@@ -3067,9 +3067,6 @@ major_do_collection (const char *reason)
        TV_GETTIME (btv);
        time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
 
-       if (xdomain_checks)
-               check_for_xdomain_refs ();
-
        nursery_section->next_data = nursery_real_end;
        /* we should also coalesce scanning from sections close to each other
         * and deal with pointers outside of the sections later.
@@ -3078,6 +3075,9 @@ major_do_collection (const char *reason)
        if (major_collector.start_major_collection)
                major_collector.start_major_collection ();
 
+       if (xdomain_checks)
+               check_for_xdomain_refs ();
+
        /* The remsets are not useful for a major collection */
        clear_remsets ();
        global_remset_cache_clear ();

diff --git a/mono/metadata/sgen-gc.h b/mono/metadata/sgen-gc.h
index c76cfd569e2b9a5173b8b84efbcfaa73b0f08d14..47ecc984f90d734beb943c053162198ae7c0d3d4 100644 (file)
--- a/mono/metadata/sgen-gc.h
+++ b/mono/metadata/sgen-gc.h
@@ -65,6 +65,11 @@ typedef guint32 mword;
 typedef guint64 mword;
 #endif
 
+#define SGEN_TV_DECLARE(name) gint64 name
+#define SGEN_TV_GETTIME(tv) tv = mono_100ns_ticks ()
+#define SGEN_TV_ELAPSED(start,end) (int)((end-start) / 10)
+#define SGEN_TV_ELAPSED_MS(start,end) ((SGEN_TV_ELAPSED((start),(end)) + 500) / 1000)
+
 /* for use with write barriers */
 typedef struct _RememberedSet RememberedSet;
 struct _RememberedSet {
@@ -707,6 +712,7 @@ struct _SgenMajorCollector {
        int (*get_num_major_sections) (void);
        gboolean (*handle_gc_param) (const char *opt);
        void (*print_gc_param_usage) (void);
+       gboolean (*is_worker_thread) (pthread_t thread);
 };
 
 void mono_sgen_marksweep_init (SgenMajorCollector *collector) MONO_INTERNAL;

diff --git a/mono/metadata/sgen-marksweep.c b/mono/metadata/sgen-marksweep.c
index d3fabaa4e82655021f51af714d69022504999dee..0d9f2a1d767d0ce5198721e05883f2cfc43a7fe6 100644 (file)
--- a/mono/metadata/sgen-marksweep.c
+++ b/mono/metadata/sgen-marksweep.c
@@ -29,8 +29,11 @@
 #ifdef HAVE_SGEN_GC
 
 #include <math.h>
+#include <errno.h>
 
 #include "utils/mono-counters.h"
+#include "utils/mono-semaphore.h"
+#include "utils/mono-time.h"
 #include "metadata/object-internals.h"
 #include "metadata/profiler-private.h"
 
@@ -219,6 +222,48 @@ static MSBlockInfo **free_block_lists [MS_BLOCK_TYPE_MAX];
 static long long stat_major_blocks_alloced = 0;
 static long long stat_major_blocks_freed = 0;
 static long long stat_major_objects_evacuated = 0;
+static long long stat_time_wait_for_sweep = 0;
+
+static gboolean ms_sweep_in_progress = FALSE;
+static pthread_t ms_sweep_thread;
+static MonoSemType ms_sweep_cmd_semaphore;
+static MonoSemType ms_sweep_done_semaphore;
+
+static void
+ms_signal_sweep_command (void)
+{
+       g_assert (!ms_sweep_in_progress);
+       ms_sweep_in_progress = TRUE;
+       MONO_SEM_POST (&ms_sweep_cmd_semaphore);
+}
+
+static void
+ms_signal_sweep_done (void)
+{
+       MONO_SEM_POST (&ms_sweep_done_semaphore);
+}
+
+static void
+ms_wait_for_sweep_done (void)
+{
+       SGEN_TV_DECLARE (atv);
+       SGEN_TV_DECLARE (btv);
+       int result;
+
+       if (!ms_sweep_in_progress)
+               return;
+
+       SGEN_TV_GETTIME (atv);
+       while ((result = MONO_SEM_WAIT (&ms_sweep_done_semaphore)) != 0) {
+               if (errno != EINTR)
+                       g_error ("MONO_SEM_WAIT");
+       }
+       SGEN_TV_GETTIME (btv);
+       stat_time_wait_for_sweep += SGEN_TV_ELAPSED_MS (atv, btv);
+
+       g_assert (ms_sweep_in_progress);
+       ms_sweep_in_progress = FALSE;
+}
 
 static int
 ms_find_block_obj_size_index (int size)
@@ -560,6 +605,8 @@ alloc_obj (int size, gboolean pinned, gboolean has_references)
 
        LOCK_MS_BLOCK_LIST;
 
+       g_assert (!ms_sweep_in_progress);
+
        if (!free_blocks [size_index]) {
                if (G_UNLIKELY (!ms_alloc_block (size_index, pinned, has_references))) {
                        UNLOCK_MS_BLOCK_LIST;
@@ -634,7 +681,11 @@ major_free_non_pinned_object (char *obj, size_t size)
 static void*
 major_alloc_small_pinned_obj (size_t size, gboolean has_references)
 {
-        void *res = alloc_obj (size, TRUE, has_references);
+       void *res;
+
+       ms_wait_for_sweep_done ();
+
+       res = alloc_obj (size, TRUE, has_references);
         /*If we failed to alloc memory, we better try releasing memory
          *as pinned alloc is requested by the runtime.
          */
@@ -658,7 +709,12 @@ static void*
 major_alloc_degraded (MonoVTable *vtable, size_t size)
 {
        void *obj;
-       int old_num_sections = num_major_sections;
+       int old_num_sections;
+
+       ms_wait_for_sweep_done ();
+
+       old_num_sections = num_major_sections;
+
        obj = alloc_obj (size, FALSE, vtable->klass->has_references);
        if (G_LIKELY (obj)) {
                *(MonoVTable**)obj = vtable;
@@ -725,6 +781,8 @@ major_iterate_objects (gboolean non_pinned, gboolean pinned, IterateObjectCallba
 {
        MSBlockInfo *block;
 
+       ms_wait_for_sweep_done ();
+
        FOREACH_BLOCK (block) {
                int count = MS_BLOCK_FREE / block->obj_size;
                int i;
@@ -1085,7 +1143,7 @@ mark_pinned_objects_in_block (MSBlockInfo *block, SgenGrayQueue *queue)
 }
 
 static void
-major_sweep (void)
+ms_sweep (void)
 {
        int i;
 #ifdef FIXED_HEAP
@@ -1222,6 +1280,31 @@ major_sweep (void)
        }
 }
 
+static void
+major_sweep (void)
+{
+       ms_signal_sweep_command ();
+}
+
+static void*
+ms_sweep_thread_func (void *dummy)
+{
+       for (;;) {
+               int result;
+
+               while ((result = MONO_SEM_WAIT (&ms_sweep_cmd_semaphore)) != 0) {
+                       if (errno != EINTR)
+                               g_error ("MONO_SEM_WAIT");
+               }
+
+               ms_sweep ();
+
+               ms_signal_sweep_done ();
+       }
+
+       return NULL;
+}
+
 static int count_pinned_ref;
 static int count_pinned_nonref;
 static int count_nonpinned_ref;
@@ -1300,6 +1383,8 @@ static int old_num_major_sections;
 static void
 major_start_nursery_collection (void)
 {
+       ms_wait_for_sweep_done ();
+
 #ifdef MARKSWEEP_CONSISTENCY_CHECK
        consistency_check ();
 #endif
@@ -1321,6 +1406,8 @@ major_start_major_collection (void)
 {
        int i;
 
+       ms_wait_for_sweep_done ();
+
        /* clear the free lists */
        for (i = 0; i < num_block_obj_sizes; ++i) {
                if (!evacuate_block_obj_sizes [i])
@@ -1598,6 +1685,14 @@ major_scan_card_table (SgenGrayQueue *queue)
 }
 #endif
 
+static gboolean
+major_is_worker_thread (pthread_t thread)
+{
+       return thread == ms_sweep_thread;
+}
+
+#undef pthread_create
+
 void
 #ifdef SGEN_PARALLEL_MARK
 #ifdef FIXED_HEAP
@@ -1650,6 +1745,11 @@ mono_sgen_marksweep_init
        mono_counters_register ("# major blocks allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_major_blocks_alloced);
        mono_counters_register ("# major blocks freed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_major_blocks_freed);
        mono_counters_register ("# major objects evacuated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_major_objects_evacuated);
+       mono_counters_register ("Wait for sweep time", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_time_wait_for_sweep);
+
+       MONO_SEM_INIT (&ms_sweep_cmd_semaphore, 0);
+       MONO_SEM_INIT (&ms_sweep_done_semaphore, 0);
+       pthread_create (&ms_sweep_thread, NULL, ms_sweep_thread_func, NULL);
 
        collector->section_size = MAJOR_SECTION_SIZE;
 #ifdef SGEN_PARALLEL_MARK
@@ -1689,6 +1789,7 @@ mono_sgen_marksweep_init
        collector->get_num_major_sections = get_num_major_sections;
        collector->handle_gc_param = major_handle_gc_param;
        collector->print_gc_param_usage = major_print_gc_param_usage;
+       collector->is_worker_thread = major_is_worker_thread;
 
        FILL_COLLECTOR_COPY_OBJECT (collector);
        FILL_COLLECTOR_SCAN_OBJECT (collector);

diff --git a/mono/metadata/sgen-workers.c b/mono/metadata/sgen-workers.c
index bdc6242ca75bd271a29cc31e3e400943f8ba73ea..8475a58ba7fb3111d962c72c45644d529e46ac66 100644 (file)
--- a/mono/metadata/sgen-workers.c
+++ b/mono/metadata/sgen-workers.c
@@ -313,6 +313,9 @@ mono_sgen_is_worker_thread (pthread_t thread)
 {
        int i;
 
+       if (major_collector.is_worker_thread && major_collector.is_worker_thread (thread))
+               return TRUE;
+
        if (!major_collector.is_parallel)
                return FALSE;