[sgen] Add free lists for each workers

These will be used for object allocation by the workers

diff --git a/mono/sgen/sgen-gc.c b/mono/sgen/sgen-gc.c
index cfe50a1afc8af0a4ea96d158dba66258c89b9a95..ea5b232afe6f42409e632b56677ceb781dcbcdd9 100644 (file)
--- a/mono/sgen/sgen-gc.c
+++ b/mono/sgen/sgen-gc.c
@@ -3227,7 +3227,7 @@ sgen_gc_init (void)
                        if (num_workers < 1)
                                num_workers = 1;
                }
-               sgen_workers_init (num_workers);
+               sgen_workers_init (num_workers, (SgenWorkerCallback) major_collector.worker_init_cb);
        }
 
        sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);

diff --git a/mono/sgen/sgen-gc.h b/mono/sgen/sgen-gc.h
index 08f1fece1cf003c68d4368f7bcee2372d089dd55..d6e320d0fb5a48faf140a8199564b89cd72a5685 100644 (file)
--- a/mono/sgen/sgen-gc.h
+++ b/mono/sgen/sgen-gc.h
@@ -677,6 +677,8 @@ struct _SgenMajorCollector {
        guint8* (*get_cardtable_mod_union_for_reference) (char *object);
        long long (*get_and_reset_num_major_objects_marked) (void);
        void (*count_cards) (long long *num_total_cards, long long *num_marked_cards);
+
+       void (*worker_init_cb) (gpointer worker);
 };
 
 extern SgenMajorCollector major_collector;

diff --git a/mono/sgen/sgen-marksweep.c b/mono/sgen/sgen-marksweep.c
index 9e160f5159deae9083e6af815eb09de24b07ce63..3b006682346fec47f4f4c7d0cd4cd7c2cf6e1aef 100644 (file)
--- a/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@ -244,6 +244,7 @@ static volatile size_t num_major_sections = 0;
  * thread only ever adds blocks to the free list, so the ABA problem can't occur.
  */
 static MSBlockInfo * volatile *free_block_lists [MS_BLOCK_TYPE_MAX];
+static MonoNativeTlsKey worker_block_free_list_key;
 
 static guint64 stat_major_blocks_alloced = 0;
 static guint64 stat_major_blocks_freed = 0;
@@ -1457,6 +1458,21 @@ static size_t *sweep_num_blocks;
 static volatile size_t num_major_sections_before_sweep;
 static volatile size_t num_major_sections_freed_in_sweep;
 
+static void
+sgen_worker_clear_free_block_lists (WorkerData *worker)
+{
+       int i, j;
+
+       if (!worker->free_block_lists)
+               return;
+
+       for (i = 0; i < MS_BLOCK_TYPE_MAX; i++) {
+               for (j = 0; j < num_block_obj_sizes; j++) {
+                       ((MSBlockInfo***) worker->free_block_lists) [i][j] = NULL;
+               }
+       }
+}
+
 static void
 sweep_start (void)
 {
@@ -1472,6 +1488,8 @@ sweep_start (void)
                for (j = 0; j < num_block_obj_sizes; ++j)
                        free_blocks [j] = NULL;
        }
+
+       sgen_workers_foreach (sgen_worker_clear_free_block_lists);
 }
 
 static void sweep_finish (void);
@@ -2595,6 +2613,22 @@ post_param_init (SgenMajorCollector *collector)
        collector->needs_thread_pool = concurrent_mark || concurrent_sweep;
 }
 
+/* We are guaranteed to be called by the worker in question */
+static void
+sgen_worker_init_callback (gpointer worker_untyped)
+{
+       int i;
+       WorkerData *worker = (WorkerData*) worker_untyped;
+       MSBlockInfo ***worker_free_blocks = (MSBlockInfo ***) sgen_alloc_internal_dynamic (sizeof (MSBlockInfo**) * MS_BLOCK_TYPE_MAX, INTERNAL_MEM_MS_TABLES, TRUE);
+
+       for (i = 0; i < MS_BLOCK_TYPE_MAX; i++)
+               worker_free_blocks [i] = (MSBlockInfo **) sgen_alloc_internal_dynamic (sizeof (MSBlockInfo*) * num_block_obj_sizes, INTERNAL_MEM_MS_TABLES, TRUE);
+
+       worker->free_block_lists = worker_free_blocks;
+
+       mono_native_tls_set_value (worker_block_free_list_key, worker_free_blocks);
+}
+
 static void
 sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurrent, gboolean is_parallel)
 {
@@ -2715,6 +2749,10 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 
                        /* FIXME use parallel obj ops */
                        collector->major_ops_conc_par_finish = collector->major_ops_concurrent_finish;
+
+                       collector->worker_init_cb = sgen_worker_init_callback;
+
+                       mono_native_tls_alloc (&worker_block_free_list_key, NULL);
                }
        }
 

diff --git a/mono/sgen/sgen-workers.c b/mono/sgen/sgen-workers.c
index 2d85a38f8972d10129cd5e9b3a6e1ec8303c596a..f6c8859bd688381d43b21ea995076d9c0913dbe2 100644 (file)
--- a/mono/sgen/sgen-workers.c
+++ b/mono/sgen/sgen-workers.c
@@ -22,6 +22,7 @@
 static int workers_num;
 static volatile gboolean forced_stop;
 static WorkerData *workers_data;
+static SgenWorkerCallback worker_init_cb;
 
 /*
  * When using multiple workers, we need to have the last worker
@@ -239,6 +240,9 @@ thread_pool_init_func (void *data_untyped)
                return;
 
        init_private_gray_queue (data);
+
+       if (worker_init_cb)
+               worker_init_cb (data);
 }
 
 static gboolean
@@ -300,7 +304,7 @@ sgen_workers_init_distribute_gray_queue (void)
 }
 
 void
-sgen_workers_init (int num_workers)
+sgen_workers_init (int num_workers, SgenWorkerCallback callback)
 {
        int i;
        void **workers_data_ptrs = (void **)alloca(num_workers * sizeof(void *));
@@ -323,6 +327,8 @@ sgen_workers_init (int num_workers)
        for (i = 0; i < num_workers; ++i)
                workers_data_ptrs [i] = (void *) &workers_data [i];
 
+       worker_init_cb = callback;
+
        sgen_thread_pool_init (num_workers, thread_pool_init_func, marker_idle_func, continue_idle_func, workers_data_ptrs);
 
        mono_counters_register ("# workers finished", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_num_finished);
@@ -451,4 +457,13 @@ sgen_workers_get_job_split_count (void)
        return (workers_num > 1) ? workers_num * 4 : 1;
 }
 
+void
+sgen_workers_foreach (SgenWorkerCallback callback)
+{
+       int i;
+
+       for (i = 0; i < workers_num; i++)
+               callback (&workers_data [i]);
+}
+
 #endif

diff --git a/mono/sgen/sgen-workers.h b/mono/sgen/sgen-workers.h
index 48dc3bb07287844e664d0ccd48d17c6cfbf3c6aa..7c6c2fc34d1e98a45b71b106efd41dd0c450cab9 100644 (file)
--- a/mono/sgen/sgen-workers.h
+++ b/mono/sgen/sgen-workers.h
@@ -12,16 +12,23 @@
 
 #include "mono/sgen/sgen-thread-pool.h"
 
-
-typedef void (*SgenWorkersFinishCallback) (void);
-
 typedef struct _WorkerData WorkerData;
 struct _WorkerData {
        gint32 state;
        SgenGrayQueue private_gray_queue; /* only read/written by worker thread */
+       /*
+        * Workers allocate major objects only from here. It has same structure as the
+        * global one. This is normally accessed from the worker_block_free_list_key.
+        * We hold it here so we can clear free lists from all threads before sweep
+        * starts.
+        */
+       gpointer free_block_lists;
 };
 
-void sgen_workers_init (int num_workers);
+typedef void (*SgenWorkersFinishCallback) (void);
+typedef void (*SgenWorkerCallback) (WorkerData *data);
+
+void sgen_workers_init (int num_workers, SgenWorkerCallback callback);
 void sgen_workers_stop_all_workers (void);
 void sgen_workers_start_all_workers (SgenObjectOperations *object_ops, SgenWorkersFinishCallback finish_job);
 void sgen_workers_init_distribute_gray_queue (void);
@@ -37,5 +44,6 @@ void sgen_workers_assert_gray_queue_is_empty (void);
 void sgen_workers_take_from_queue_and_awake (SgenGrayQueue *queue);
 SgenObjectOperations* sgen_workers_get_idle_func_object_ops (void);
 int sgen_workers_get_job_split_count (void);
+void sgen_workers_foreach (SgenWorkerCallback callback);
 
 #endif