#include "sgen/sgen-client.h"
#include "sgen/sgen-cardtable.h"
#include "sgen/sgen-pinning.h"
-#include "sgen/sgen-thread-pool.h"
+#include "sgen/sgen-workers.h"
#include "metadata/marshal.h"
#include "metadata/method-builder.h"
#include "metadata/abi-details.h"
* lock-free data structure for the queue as multiple threads will be
* adding to it at the same time.
*/
- if (sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ())) {
+ if (sgen_workers_is_worker_thread (mono_native_thread_id_get ())) {
sgen_pointer_queue_add (&moved_objects_queue, obj);
sgen_pointer_queue_add (&moved_objects_queue, destination);
} else {
void
mono_gc_base_cleanup (void)
{
- sgen_thread_pool_shutdown ();
+ sgen_thread_pool_shutdown (major_collector.get_sweep_pool ());
+
+ sgen_workers_shutdown ();
// We should have consumed any outstanding moves.
g_assert (sgen_pointer_queue_is_empty (&moved_objects_queue));
#include "sgen/sgen-gc.h"
#include "sgen/sgen-protocol.h"
#include "sgen/sgen-memory-governor.h"
-#include "sgen/sgen-thread-pool.h"
+#include "sgen/sgen-workers.h"
#include "metadata/profiler-private.h"
#include "sgen/sgen-client.h"
#include "metadata/sgen-bridge-internals.h"
We can't suspend the workers that will do all the heavy lifting.
FIXME Use some state bit in SgenThreadInfo for this.
*/
- if (sgen_thread_pool_is_thread_pool_thread (mono_thread_info_get_tid (info))) {
+ if (sgen_thread_pool_is_thread_pool_thread (major_collector.get_sweep_pool (), mono_thread_info_get_tid (info)) ||
+ sgen_workers_is_worker_thread (mono_thread_info_get_tid (info))) {
if (reason)
*reason = 4;
return FALSE;
static guint64 large_objects;
static guint64 bloby_objects;
#endif
-static guint64 major_card_scan_time;
-static guint64 los_card_scan_time;
-
-static guint64 last_major_scan_time;
-static guint64 last_los_scan_time;
-
-static void sgen_card_tables_collect_stats (gboolean begin);
mword
sgen_card_table_number_of_cards_in_range (mword address, mword size)
}
static void
-sgen_card_table_finish_minor_collection (void)
-{
- sgen_card_tables_collect_stats (FALSE);
-}
-
-static void
-sgen_card_table_scan_remsets (ScanCopyContext ctx)
+sgen_card_table_start_scan_remsets (void)
{
- SGEN_TV_DECLARE (atv);
- SGEN_TV_DECLARE (btv);
-
- sgen_card_tables_collect_stats (TRUE);
-
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
/*FIXME we should have a bit on each block/los object telling if the object have marked cards.*/
/*First we copy*/
/*Then we clear*/
sgen_card_table_clear_cards ();
#endif
- SGEN_TV_GETTIME (atv);
- sgen_get_major_collector ()->scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, 0, 1);
- SGEN_TV_GETTIME (btv);
- last_major_scan_time = SGEN_TV_ELAPSED (atv, btv);
- major_card_scan_time += last_major_scan_time;
- sgen_los_scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, 0, 1);
- SGEN_TV_GETTIME (atv);
- last_los_scan_time = SGEN_TV_ELAPSED (btv, atv);
- los_card_scan_time += last_los_scan_time;
-
- sgen_wbroots_scan_card_table (ctx);
}
guint8*
binary_protocol_card_scan (obj, sgen_safe_object_get_size (obj));
}
-#ifdef CARDTABLE_STATS
-
-typedef struct {
- int total, marked, remarked, gc_marked;
-} card_stats;
-
-static card_stats major_stats, los_stats;
-static card_stats *cur_stats;
-
-static void
-count_marked_cards (mword start, mword size)
-{
- mword end = start + size;
- while (start <= end) {
- guint8 card = *sgen_card_table_get_card_address (start);
- ++cur_stats->total;
- if (card)
- ++cur_stats->marked;
- if (card == 2)
- ++cur_stats->gc_marked;
- start += CARD_SIZE_IN_BYTES;
- }
-}
-
-static void
-count_remarked_cards (mword start, mword size)
-{
- mword end = start + size;
- while (start <= end) {
- if (sgen_card_table_address_is_marked (start)) {
- ++cur_stats->remarked;
- *sgen_card_table_get_card_address (start) = 2;
- }
- start += CARD_SIZE_IN_BYTES;
- }
-}
-
-#endif
-
-static void
-sgen_card_tables_collect_stats (gboolean begin)
-{
-#ifdef CARDTABLE_STATS
- if (begin) {
- memset (&major_stats, 0, sizeof (card_stats));
- memset (&los_stats, 0, sizeof (card_stats));
- cur_stats = &major_stats;
- sgen_major_collector_iterate_live_block_ranges (count_marked_cards);
- cur_stats = &los_stats;
- sgen_los_iterate_live_block_ranges (count_marked_cards);
- } else {
- cur_stats = &major_stats;
- sgen_major_collector_iterate_live_block_ranges (count_remarked_cards);
- cur_stats = &los_stats;
- sgen_los_iterate_live_block_ranges (count_remarked_cards);
- printf ("cards major (t %d m %d g %d r %d) los (t %d m %d g %d r %d) major_scan %.2fms los_scan %.2fms\n",
- major_stats.total, major_stats.marked, major_stats.gc_marked, major_stats.remarked,
- los_stats.total, los_stats.marked, los_stats.gc_marked, los_stats.remarked,
- last_major_scan_time / 10000.0f, last_los_scan_time / 10000.0f);
- }
-#endif
-}
-
void
sgen_card_table_init (SgenRememberedSet *remset)
{
mono_counters_register ("cardtable large objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &large_objects);
mono_counters_register ("cardtable bloby objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &bloby_objects);
#endif
- mono_counters_register ("cardtable major scan time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &major_card_scan_time);
- mono_counters_register ("cardtable los scan time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &los_card_scan_time);
-
remset->wbarrier_set_field = sgen_card_table_wbarrier_set_field;
remset->wbarrier_arrayref_copy = sgen_card_table_wbarrier_arrayref_copy;
remset->wbarrier_generic_nostore = sgen_card_table_wbarrier_generic_nostore;
remset->record_pointer = sgen_card_table_record_pointer;
- remset->scan_remsets = sgen_card_table_scan_remsets;
+ remset->start_scan_remsets = sgen_card_table_start_scan_remsets;
- remset->finish_minor_collection = sgen_card_table_finish_minor_collection;
remset->clear_cards = sgen_card_table_clear_cards;
remset->find_address = sgen_card_table_find_address;
GRAY_OBJECT_ENQUEUE_PARALLEL (queue, (GCObject *)destination, sgen_vtable_get_descriptor (vt));
}
} else {
+ /*
+ * Unlikely case. Clear the allocated object so it doesn't confuse nursery
+ * card table scanning, since it can contain old invalid refs.
+ * FIXME make sure it is not a problem if another threads scans it while we clear
+ */
+ mono_gc_bzero_aligned (destination, objsize);
destination = final_destination;
}
}
static guint64 time_minor_pre_collection_fragment_clear = 0;
static guint64 time_minor_pinning = 0;
static guint64 time_minor_scan_remsets = 0;
+static guint64 time_minor_scan_major_blocks = 0;
+static guint64 time_minor_scan_los = 0;
static guint64 time_minor_scan_pinned = 0;
static guint64 time_minor_scan_roots = 0;
static guint64 time_minor_finish_gray_stack = 0;
static void
gray_queue_redirect (SgenGrayQueue *queue)
{
- SGEN_ASSERT (0, concurrent_collection_in_progress, "Where are we redirecting the gray queue to, without a concurrent collection?");
-
sgen_workers_take_from_queue (queue);
}
mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_pre_collection_fragment_clear);
mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_pinning);
mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_remsets);
+ mono_counters_register ("Minor scan major blocks", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_major_blocks);
+ mono_counters_register ("Minor scan los", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_los);
mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_pinned);
mono_counters_register ("Minor scan roots", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_roots);
mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_fragment_creation);
return CONTEXT_FROM_OBJECT_OPERATIONS (job->ops, sgen_workers_get_job_gray_queue (worker_data, job->gc_thread_gray_queue));
}
-static void
-job_remembered_set_scan (void *worker_data_untyped, SgenThreadPoolJob *job)
-{
- remset.scan_remsets (scan_copy_context_for_scan_job (worker_data_untyped, (ScanJob*)job));
-}
-
typedef struct {
ScanJob scan_job;
char *heap_start;
scan_finalizer_entries (job_data->queue, ctx);
}
+static void
+job_scan_wbroots (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ ScanJob *job_data = (ScanJob*)job;
+ ScanCopyContext ctx = scan_copy_context_for_scan_job (worker_data_untyped, job_data);
+
+ sgen_wbroots_scan_card_table (ctx);
+}
+
+static void
+job_scan_major_card_table (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ SGEN_TV_DECLARE (atv);
+ SGEN_TV_DECLARE (btv);
+ ParallelScanJob *job_data = (ParallelScanJob*)job;
+ ScanCopyContext ctx = scan_copy_context_for_scan_job (worker_data_untyped, (ScanJob*)job_data);
+
+ SGEN_TV_GETTIME (atv);
+ major_collector.scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, job_data->job_index, sgen_workers_get_job_split_count ());
+ SGEN_TV_GETTIME (btv);
+ time_minor_scan_major_blocks += SGEN_TV_ELAPSED (atv, btv);
+}
+
+static void
+job_scan_los_card_table (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ SGEN_TV_DECLARE (atv);
+ SGEN_TV_DECLARE (btv);
+ ParallelScanJob *job_data = (ParallelScanJob*)job;
+ ScanCopyContext ctx = scan_copy_context_for_scan_job (worker_data_untyped, (ScanJob*)job_data);
+
+ SGEN_TV_GETTIME (atv);
+ sgen_los_scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, job_data->job_index, sgen_workers_get_job_split_count ());
+ SGEN_TV_GETTIME (btv);
+ time_minor_scan_los += SGEN_TV_ELAPSED (atv, btv);
+}
+
static void
job_scan_major_mod_union_card_table (void *worker_data_untyped, SgenThreadPoolJob *job)
{
sgen_gray_object_queue_init (gc_thread_gray_queue, NULL, TRUE);
}
+static void
+enqueue_scan_remembered_set_jobs (SgenGrayQueue *gc_thread_gray_queue, SgenObjectOperations *ops, gboolean enqueue)
+{
+ int i, split_count = sgen_workers_get_job_split_count ();
+ ScanJob *sj;
+
+ sj = (ScanJob*)sgen_thread_pool_job_alloc ("scan wbroots", job_scan_wbroots, sizeof (ScanJob));
+ sj->ops = ops;
+ sj->gc_thread_gray_queue = gc_thread_gray_queue;
+ sgen_workers_enqueue_job (&sj->job, enqueue);
+
+ for (i = 0; i < split_count; i++) {
+ ParallelScanJob *psj;
+
+ psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan major remsets", job_scan_major_card_table, sizeof (ParallelScanJob));
+ psj->scan_job.ops = ops;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
+ psj->job_index = i;
+ sgen_workers_enqueue_job (&psj->scan_job.job, enqueue);
+
+ psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan LOS remsets", job_scan_los_card_table, sizeof (ParallelScanJob));
+ psj->scan_job.ops = ops;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
+ psj->job_index = i;
+ sgen_workers_enqueue_job (&psj->scan_job.job, enqueue);
+ }
+}
+
static void
enqueue_scan_from_roots_jobs (SgenGrayQueue *gc_thread_gray_queue, char *heap_start, char *heap_end, SgenObjectOperations *ops, gboolean enqueue)
{
static gboolean
collect_nursery (const char *reason, gboolean is_overflow, SgenGrayQueue *unpin_queue)
{
- gboolean needs_major;
+ gboolean needs_major, is_parallel = FALSE;
size_t max_garbage_amount;
char *nursery_next;
mword fragment_total;
- ScanJob *sj;
SgenGrayQueue gc_thread_gray_queue;
- SgenObjectOperations *object_ops;
+ SgenObjectOperations *object_ops_nopar, *object_ops_par = NULL;
ScanCopyContext ctx;
TV_DECLARE (atv);
TV_DECLARE (btv);
binary_protocol_collection_begin (gc_stats.minor_gc_count, GENERATION_NURSERY);
- if (sgen_concurrent_collection_in_progress ())
- object_ops = &sgen_minor_collector.serial_ops_with_concurrent_major;
- else
- object_ops = &sgen_minor_collector.serial_ops;
+ if (sgen_concurrent_collection_in_progress ()) {
+ /* FIXME Support parallel nursery collections with concurrent major */
+ object_ops_nopar = &sgen_minor_collector.serial_ops_with_concurrent_major;
+ } else {
+ object_ops_nopar = &sgen_minor_collector.serial_ops;
+ if (sgen_minor_collector.is_parallel) {
+ object_ops_par = &sgen_minor_collector.parallel_ops;
+ is_parallel = TRUE;
+ }
+ }
if (do_verify_nursery || do_dump_nursery_content)
sgen_debug_verify_nursery (do_dump_nursery_content);
sgen_memgov_minor_collection_start ();
- init_gray_queue (&gc_thread_gray_queue, FALSE);
- ctx = CONTEXT_FROM_OBJECT_OPERATIONS (object_ops, &gc_thread_gray_queue);
+ init_gray_queue (&gc_thread_gray_queue, is_parallel);
+ ctx = CONTEXT_FROM_OBJECT_OPERATIONS (object_ops_nopar, &gc_thread_gray_queue);
gc_stats.minor_gc_count ++;
SGEN_LOG (2, "Finding pinned pointers: %zd in %lld usecs", sgen_get_pinned_count (), (long long)TV_ELAPSED (btv, atv));
SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
- sj = (ScanJob*)sgen_thread_pool_job_alloc ("scan remset", job_remembered_set_scan, sizeof (ScanJob));
- sj->ops = object_ops;
- sj->gc_thread_gray_queue = &gc_thread_gray_queue;
- sgen_workers_enqueue_job (&sj->job, FALSE);
+ remset.start_scan_remsets ();
+
+ enqueue_scan_remembered_set_jobs (&gc_thread_gray_queue, is_parallel ? object_ops_par : object_ops_nopar, is_parallel);
/* we don't have complete write barrier yet, so we scan all the old generation sections */
TV_GETTIME (btv);
TV_GETTIME (atv);
time_minor_scan_pinned += TV_ELAPSED (btv, atv);
- enqueue_scan_from_roots_jobs (&gc_thread_gray_queue, sgen_get_nursery_start (), nursery_next, object_ops, FALSE);
+ enqueue_scan_from_roots_jobs (&gc_thread_gray_queue, sgen_get_nursery_start (), nursery_next, is_parallel ? object_ops_par : object_ops_nopar, is_parallel);
+
+ if (is_parallel) {
+ gray_queue_redirect (&gc_thread_gray_queue);
+ sgen_workers_start_all_workers (object_ops_nopar, object_ops_par, NULL);
+ sgen_workers_join ();
+ }
TV_GETTIME (btv);
time_minor_scan_roots += TV_ELAPSED (atv, btv);
sgen_gray_object_queue_dispose (&gc_thread_gray_queue);
- remset.finish_minor_collection ();
-
check_scan_starts ();
binary_protocol_flush_buffers (FALSE);
SGEN_ASSERT (0, sgen_workers_all_done (), "Why are the workers not done when we start or finish a major collection?");
if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) {
- sgen_workers_set_num_active_workers (0);
+ if (object_ops_par != NULL)
+ sgen_workers_set_num_active_workers (0);
if (sgen_workers_have_idle_work ()) {
/*
* We force the finish of the worker with the new object ops context
if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) {
int i, split_count = sgen_workers_get_job_split_count ();
+ gboolean parallel = object_ops_par != NULL;
- gray_queue_redirect (gc_thread_gray_queue);
+ /* If we're not parallel we finish the collection on the gc thread */
+ if (parallel)
+ gray_queue_redirect (gc_thread_gray_queue);
/* Mod union card table */
for (i = 0; i < split_count; i++) {
psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan mod union cardtable", job_scan_major_mod_union_card_table, sizeof (ParallelScanJob));
psj->scan_job.ops = object_ops_par ? object_ops_par : object_ops_nopar;
- psj->scan_job.gc_thread_gray_queue = NULL;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
psj->job_index = i;
- sgen_workers_enqueue_job (&psj->scan_job.job, TRUE);
+ sgen_workers_enqueue_job (&psj->scan_job.job, parallel);
psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan LOS mod union cardtable", job_scan_los_mod_union_card_table, sizeof (ParallelScanJob));
psj->scan_job.ops = object_ops_par ? object_ops_par : object_ops_nopar;
- psj->scan_job.gc_thread_gray_queue = NULL;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
psj->job_index = i;
- sgen_workers_enqueue_job (&psj->scan_job.job, TRUE);
+ sgen_workers_enqueue_job (&psj->scan_job.job, parallel);
}
- /*
- * If we enqueue a job while workers are running we need to sgen_workers_ensure_awake
- * in order to make sure that we are running the idle func and draining all worker
- * gray queues. The operation of starting workers implies this, so we start them after
- * in order to avoid doing this operation twice. The workers will drain the main gray
- * stack that contained roots and pinned objects and also scan the mod union card
- * table.
- */
- sgen_workers_start_all_workers (object_ops_nopar, object_ops_par, NULL);
- sgen_workers_join ();
+ if (parallel) {
+ /*
+ * If we enqueue a job while workers are running we need to sgen_workers_ensure_awake
+ * in order to make sure that we are running the idle func and draining all worker
+ * gray queues. The operation of starting workers implies this, so we start them after
+ * in order to avoid doing this operation twice. The workers will drain the main gray
+ * stack that contained roots and pinned objects and also scan the mod union card
+ * table.
+ */
+ sgen_workers_start_all_workers (object_ops_nopar, object_ops_par, NULL);
+ sgen_workers_join ();
+ }
}
sgen_pin_stats_report ();
sgen_client_init ();
if (!minor_collector_opt) {
- sgen_simple_nursery_init (&sgen_minor_collector);
+ sgen_simple_nursery_init (&sgen_minor_collector, FALSE);
} else {
if (!strcmp (minor_collector_opt, "simple")) {
use_simple_nursery:
- sgen_simple_nursery_init (&sgen_minor_collector);
+ sgen_simple_nursery_init (&sgen_minor_collector, FALSE);
+ } else if (!strcmp (minor_collector_opt, "simple-par")) {
+ sgen_simple_nursery_init (&sgen_minor_collector, TRUE);
} else if (!strcmp (minor_collector_opt, "split")) {
sgen_split_nursery_init (&sgen_minor_collector);
} else {
if (major_collector.post_param_init)
major_collector.post_param_init (&major_collector);
- if (major_collector.needs_thread_pool) {
+ if (major_collector.is_concurrent || sgen_minor_collector.is_parallel) {
int num_workers = 1;
- if (major_collector.is_parallel) {
+ if (major_collector.is_parallel || sgen_minor_collector.is_parallel) {
/* FIXME Detect the number of physical cores, instead of logical */
num_workers = mono_cpu_count () / 2;
if (num_workers < 1)
return &major_collector;
}
+SgenMinorCollector*
+sgen_get_minor_collector (void)
+{
+ return &sgen_minor_collector;
+}
+
SgenRememberedSet*
sgen_get_remset (void)
{
#include "mono/sgen/sgen-hash-table.h"
#include "mono/sgen/sgen-protocol.h"
#include "mono/sgen/gc-internal-agnostic.h"
+#include "mono/sgen/sgen-thread-pool.h"
/* The method used to clear the nursery */
/* Clearing at nursery collections is the safest, but has bad interactions with caches.
typedef struct {
gboolean is_split;
+ gboolean is_parallel;
GCObject* (*alloc_for_promotion) (GCVTable vtable, GCObject *obj, size_t objsize, gboolean has_references);
SgenObjectOperations serial_ops;
SgenObjectOperations serial_ops_with_concurrent_major;
+ SgenObjectOperations parallel_ops;
void (*prepare_to_space) (char *to_space_bitmap, size_t space_bitmap_size);
void (*clear_fragments) (void);
extern SgenMinorCollector sgen_minor_collector;
-void sgen_simple_nursery_init (SgenMinorCollector *collector);
+void sgen_simple_nursery_init (SgenMinorCollector *collector, gboolean parallel);
void sgen_split_nursery_init (SgenMinorCollector *collector);
/* Updating references */
{
if (!allow_null)
SGEN_ASSERT (0, o, "Cannot update a reference with a NULL pointer");
- SGEN_ASSERT (0, !sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "Can't update a reference in the worker thread");
+ SGEN_ASSERT (0, !sgen_workers_is_worker_thread (mono_native_thread_id_get ()), "Can't update a reference in the worker thread");
*p = o;
}
size_t section_size;
gboolean is_concurrent;
gboolean is_parallel;
- gboolean needs_thread_pool;
gboolean supports_cardtable;
gboolean sweeps_lazily;
guint8* (*get_cardtable_mod_union_for_reference) (char *object);
long long (*get_and_reset_num_major_objects_marked) (void);
void (*count_cards) (long long *num_total_cards, long long *num_marked_cards);
+ SgenThreadPool* (*get_sweep_pool) (void);
void (*worker_init_cb) (gpointer worker);
};
void sgen_marksweep_conc_init (SgenMajorCollector *collector);
void sgen_marksweep_conc_par_init (SgenMajorCollector *collector);
SgenMajorCollector* sgen_get_major_collector (void);
+SgenMinorCollector* sgen_get_minor_collector (void);
typedef struct _SgenRememberedSet {
void (*wbarrier_generic_nostore) (gpointer ptr);
void (*record_pointer) (gpointer ptr);
- void (*scan_remsets) (ScanCopyContext ctx);
+ void (*start_scan_remsets) (void);
void (*clear_cards) (void);
- void (*finish_minor_collection) (void);
gboolean (*find_address) (char *addr);
gboolean (*find_address_with_cards) (char *cards_start, guint8 *cards, char *addr);
} SgenRememberedSet;
#define SGEN_GRAY_QUEUE_HEADER_SIZE 3
#endif
-#define SGEN_GRAY_QUEUE_SECTION_SIZE (128 - SGEN_GRAY_QUEUE_HEADER_SIZE)
+#define SGEN_GRAY_QUEUE_SECTION_SIZE (512 - SGEN_GRAY_QUEUE_HEADER_SIZE)
#ifdef SGEN_CHECK_GRAY_OBJECT_SECTIONS
typedef enum {
static gboolean concurrent_mark;
static gboolean concurrent_sweep = TRUE;
+SgenThreadPool sweep_pool_inst;
+SgenThreadPool *sweep_pool;
+
#define BLOCK_IS_TAGGED_HAS_REFERENCES(bl) SGEN_POINTER_IS_TAGGED_1 ((bl))
#define BLOCK_TAG_HAS_REFERENCES(bl) SGEN_POINTER_TAG_1 ((bl))
*/
if (SGEN_CAS_PTR ((volatile gpointer *)&free_blocks [size_index], next_free, block) != block)
goto get_block;
- g_assert (block->free_list);
block->next_free = free_blocks_local [size_index];
free_blocks_local [size_index] = block;
wait:
job = sweep_job;
if (job)
- sgen_thread_pool_job_wait (job);
+ sgen_thread_pool_job_wait (sweep_pool, job);
SGEN_ASSERT (0, !sweep_job, "Why did the sweep job not null itself?");
SGEN_ASSERT (0, sweep_state == SWEEP_STATE_SWEPT, "How is the sweep job done but we're not swept?");
}
}
}
+static void
+sgen_worker_clear_free_block_lists_evac (WorkerData *worker)
+{
+ int i, j;
+
+ if (!worker->free_block_lists)
+ return;
+
+ for (i = 0; i < MS_BLOCK_TYPE_MAX; i++) {
+ for (j = 0; j < num_block_obj_sizes; j++) {
+ if (((MSBlockInfo***) worker->free_block_lists) [i][j])
+ SGEN_ASSERT (0, !((MSBlockInfo***) worker->free_block_lists) [i][j]->next_free, "Why do we have linked free blocks on the workers");
+
+ if (evacuate_block_obj_sizes [j])
+ ((MSBlockInfo***) worker->free_block_lists) [i][j] = NULL;
+ }
+ }
+}
+
static void
sweep_start (void)
{
*/
if (concurrent_sweep && lazy_sweep) {
sweep_blocks_job = sgen_thread_pool_job_alloc ("sweep_blocks", sweep_blocks_job_func, sizeof (SgenThreadPoolJob));
- sgen_thread_pool_job_enqueue (sweep_blocks_job);
+ sgen_thread_pool_job_enqueue (sweep_pool, sweep_blocks_job);
}
sweep_finish ();
SGEN_ASSERT (0, !sweep_job, "We haven't finished the last sweep?");
if (concurrent_sweep) {
sweep_job = sgen_thread_pool_job_alloc ("sweep", sweep_job_func, sizeof (SgenThreadPoolJob));
- sgen_thread_pool_job_enqueue (sweep_job);
+ sgen_thread_pool_job_enqueue (sweep_pool, sweep_job);
} else {
sweep_job_func (NULL, NULL);
}
sgen_evacuation_freelist_blocks (&free_block_lists [MS_BLOCK_FLAG_REFS][i], i);
}
+ /* We expect workers to have very few blocks on the freelist, just evacuate them */
+ sgen_workers_foreach (sgen_worker_clear_free_block_lists_evac);
+
if (lazy_sweep && concurrent_sweep) {
/*
* sweep_blocks_job is created before sweep_finish, which we wait for above
*/
SgenThreadPoolJob *job = sweep_blocks_job;
if (job)
- sgen_thread_pool_job_wait (job);
+ sgen_thread_pool_job_wait (sweep_pool, job);
}
if (lazy_sweep && !concurrent_sweep)
#endif
}
+static SgenThreadPool*
+major_get_sweep_pool (void)
+{
+ return sweep_pool;
+}
+
static int
compare_pointers (const void *va, const void *vb) {
char *a = *(char**)va, *b = *(char**)vb;
post_param_init (SgenMajorCollector *collector)
{
collector->sweeps_lazily = lazy_sweep;
- collector->needs_thread_pool = concurrent_mark || concurrent_sweep;
}
/* We are guaranteed to be called by the worker in question */
mono_native_tls_set_value (worker_block_free_list_key, worker_free_blocks);
}
+static void
+thread_pool_init_func (void *data_untyped)
+{
+ sgen_client_thread_register_worker ();
+}
+
static void
sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurrent, gboolean is_parallel)
{
for (i = 0; i < MS_NUM_FAST_BLOCK_OBJ_SIZE_INDEXES * 8; ++i)
g_assert (MS_BLOCK_OBJ_SIZE_INDEX (i) == ms_find_block_obj_size_index (i));
+ /* We can do this because we always init the minor before the major */
+ if (is_parallel || sgen_get_minor_collector ()->is_parallel) {
+ mono_native_tls_alloc (&worker_block_free_list_key, NULL);
+ collector->worker_init_cb = sgen_worker_init_callback;
+ }
+
mono_counters_register ("# major blocks allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_alloced);
mono_counters_register ("# major blocks freed", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed);
mono_counters_register ("# major blocks lazy swept", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_lazy_swept);
concurrent_mark = is_concurrent;
collector->is_concurrent = is_concurrent;
collector->is_parallel = is_parallel;
- collector->needs_thread_pool = is_concurrent || concurrent_sweep;
collector->get_and_reset_num_major_objects_marked = major_get_and_reset_num_major_objects_marked;
collector->supports_cardtable = TRUE;
collector->is_valid_object = major_is_valid_object;
collector->describe_pointer = major_describe_pointer;
collector->count_cards = major_count_cards;
+ collector->get_sweep_pool = major_get_sweep_pool;
collector->major_ops_serial.copy_or_mark_object = major_copy_or_mark_object_canonical;
collector->major_ops_serial.scan_object = major_scan_object_with_evacuation;
collector->major_ops_conc_par_finish.scan_vtype = major_scan_vtype_par_with_evacuation;
collector->major_ops_conc_par_finish.scan_ptr_field = major_scan_ptr_field_par_with_evacuation;
collector->major_ops_conc_par_finish.drain_gray_stack = drain_gray_stack_par;
-
- collector->worker_init_cb = sgen_worker_init_callback;
-
- mono_native_tls_alloc (&worker_block_free_list_key, NULL);
}
}
/*cardtable requires major pages to be 8 cards aligned*/
g_assert ((MS_BLOCK_SIZE % (8 * CARD_SIZE_IN_BYTES)) == 0);
+
+ if (concurrent_sweep) {
+ SgenThreadPool **thread_datas = &sweep_pool;
+ sweep_pool = &sweep_pool_inst;
+ sgen_thread_pool_init (sweep_pool, 1, thread_pool_init_func, NULL, NULL, NULL, (SgenThreadPoolData**)&thread_datas);
+ }
}
void
#include "mono/sgen/sgen-gc.h"
#include "mono/sgen/sgen-memory-governor.h"
-#include "mono/sgen/sgen-thread-pool.h"
+#include "mono/sgen/sgen-workers.h"
#include "mono/sgen/sgen-client.h"
#define MIN_MINOR_COLLECTION_ALLOWANCE ((mword)(DEFAULT_NURSERY_SIZE * default_allowance_nursery_size_ratio))
sgen_memgov_try_alloc_space (mword size, int space)
{
if (sgen_memgov_available_free_space () < size) {
- SGEN_ASSERT (4, !sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "Memory shouldn't run out in worker thread");
+ SGEN_ASSERT (4, !sgen_workers_is_worker_thread (mono_native_thread_id_get ()), "Memory shouldn't run out in worker thread");
return FALSE;
}
#if defined(SGEN_SIMPLE_NURSERY)
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+/* Not supported with concurrent major yet */
+#define SERIAL_COPY_OBJECT simple_par_nursery_copy_object
+#define SERIAL_COPY_OBJECT_FROM_OBJ simple_par_nursery_copy_object_from_obj
+#else
#ifdef SGEN_CONCURRENT_MAJOR
#define SERIAL_COPY_OBJECT simple_nursery_serial_with_concurrent_major_copy_object
#define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_with_concurrent_major_copy_object_from_obj
#define SERIAL_COPY_OBJECT simple_nursery_serial_copy_object
#define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_copy_object_from_obj
#endif
+#endif
#elif defined (SGEN_SPLIT_NURSERY)
HEAVY_STAT (++stat_objects_copied_nursery);
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+ copy = copy_object_no_checks_par (obj, queue);
+#else
copy = copy_object_no_checks (obj, queue);
+#endif
SGEN_UPDATE_REFERENCE (obj_slot, copy);
}
HEAVY_STAT (++stat_objects_copied_nursery);
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+ copy = copy_object_no_checks_par (obj, queue);
+#else
copy = copy_object_no_checks (obj, queue);
+#endif
#ifdef SGEN_CONCURRENT_MAJOR
/*
* If an object is evacuated to the major heap and a reference to it, from the major
#if defined(SGEN_SIMPLE_NURSERY)
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+#define SERIAL_SCAN_OBJECT simple_par_nursery_serial_scan_object
+#define SERIAL_SCAN_VTYPE simple_par_nursery_serial_scan_vtype
+#define SERIAL_SCAN_PTR_FIELD simple_par_nursery_serial_scan_ptr_field
+#define SERIAL_DRAIN_GRAY_STACK simple_par_nursery_serial_drain_gray_stack
+#else
#ifdef SGEN_CONCURRENT_MAJOR
#define SERIAL_SCAN_OBJECT simple_nursery_serial_with_concurrent_major_scan_object
#define SERIAL_SCAN_VTYPE simple_nursery_serial_with_concurrent_major_scan_vtype
#define SERIAL_SCAN_PTR_FIELD simple_nursery_serial_scan_ptr_field
#define SERIAL_DRAIN_GRAY_STACK simple_nursery_serial_drain_gray_stack
#endif
+#endif
#elif defined (SGEN_SPLIT_NURSERY)
static gboolean
SERIAL_DRAIN_GRAY_STACK (SgenGrayQueue *queue)
{
- for (;;) {
- GCObject *obj;
- SgenDescriptor desc;
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+ int i;
+ /*
+ * We do bounded iteration so we can switch to optimized context
+ * when we are the last worker remaining.
+ */
+ for (i = 0; i < 32; i++) {
+#else
+ for (;;) {
+#endif
+ GCObject *obj;
+ SgenDescriptor desc;
+
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+ GRAY_OBJECT_DEQUEUE_PARALLEL (queue, &obj, &desc);
+#else
+ GRAY_OBJECT_DEQUEUE_SERIAL (queue, &obj, &desc);
+#endif
+ if (!obj)
+ return TRUE;
- GRAY_OBJECT_DEQUEUE_SERIAL (queue, &obj, &desc);
- if (!obj)
- return TRUE;
+ SERIAL_SCAN_OBJECT (obj, desc, queue);
+ }
- SERIAL_SCAN_OBJECT (obj, desc, queue);
- }
+ return FALSE;
}
#define FILL_MINOR_COLLECTOR_SCAN_OBJECT(ops) do { \
#include "sgen-gc.h"
#include "sgen-protocol.h"
#include "sgen-memory-governor.h"
-#include "sgen-thread-pool.h"
+#include "sgen-workers.h"
#include "sgen-client.h"
#include "mono/utils/mono-membar.h"
#include "mono/utils/mono-proclib.h"
buffer->buffer [index++] = type;
/* We should never change the header format */
if (include_worker_index) {
+ int worker_index;
+ MonoNativeThreadId tid = mono_native_thread_id_get ();
/*
* If the thread is not a worker thread we insert 0, which is interpreted
* as gc thread. Worker indexes are 1 based.
*/
- buffer->buffer [index++] = (unsigned char) sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ());
+ worker_index = sgen_workers_is_worker_thread (tid);
+ if (!worker_index)
+ worker_index = sgen_thread_pool_is_thread_pool_thread (major_collector.get_sweep_pool (), tid);
+ /* FIXME Consider using different index bases for different thread pools */
+ buffer->buffer [index++] = (unsigned char) worker_index;
}
memcpy (buffer->buffer + index, data, size);
index += size;
#define collector_pin_object(obj, queue) sgen_pin_object (obj, queue);
#define COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION alloc_for_promotion
+#define COPY_OR_MARK_PARALLEL
#include "sgen-copy-object.h"
#define SGEN_SIMPLE_NURSERY
FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
}
+#define SGEN_SIMPLE_PAR_NURSERY
+
+#include "sgen-minor-copy-object.h"
+#include "sgen-minor-scan-object.h"
+
+static void
+fill_parallel_ops (SgenObjectOperations *ops)
+{
+ ops->copy_or_mark_object = SERIAL_COPY_OBJECT;
+ FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
+}
+
+#undef SGEN_SIMPLE_PAR_NURSERY
#define SGEN_CONCURRENT_MAJOR
#include "sgen-minor-copy-object.h"
}
void
-sgen_simple_nursery_init (SgenMinorCollector *collector)
+sgen_simple_nursery_init (SgenMinorCollector *collector, gboolean parallel)
{
collector->is_split = FALSE;
+ collector->is_parallel = parallel;
collector->alloc_for_promotion = alloc_for_promotion;
fill_serial_ops (&collector->serial_ops);
fill_serial_with_concurrent_major_ops (&collector->serial_ops_with_concurrent_major);
+ fill_parallel_ops (&collector->parallel_ops);
}
sgen_split_nursery_init (SgenMinorCollector *collector)
{
collector->is_split = TRUE;
+ collector->is_parallel = FALSE;
collector->alloc_for_promotion = minor_alloc_for_promotion;
#include "mono/sgen/sgen-gc.h"
#include "mono/sgen/sgen-thread-pool.h"
-#include "mono/sgen/sgen-pointer-queue.h"
#include "mono/utils/mono-os-mutex.h"
-#ifndef SGEN_WITHOUT_MONO
-#include "mono/utils/mono-threads.h"
-#endif
-
-#define MAX_NUM_THREADS 8
-
-static mono_mutex_t lock;
-static mono_cond_t work_cond;
-static mono_cond_t done_cond;
-
-static int threads_num = 0;
-static MonoNativeThreadId threads [MAX_NUM_THREADS];
-
-/* Only accessed with the lock held. */
-static SgenPointerQueue job_queue;
-
-static SgenThreadPoolThreadInitFunc thread_init_func;
-static SgenThreadPoolIdleJobFunc idle_job_func;
-static SgenThreadPoolContinueIdleJobFunc continue_idle_job_func;
-static SgenThreadPoolShouldWorkFunc should_work_func;
-
-static volatile gboolean threadpool_shutdown;
-static volatile int threads_finished = 0;
enum {
STATE_WAITING,
/* Assumes that the lock is held. */
static SgenThreadPoolJob*
-get_job_and_set_in_progress (void)
+get_job_and_set_in_progress (SgenThreadPool *pool)
{
- for (size_t i = 0; i < job_queue.next_slot; ++i) {
- SgenThreadPoolJob *job = (SgenThreadPoolJob *)job_queue.data [i];
+ for (size_t i = 0; i < pool->job_queue.next_slot; ++i) {
+ SgenThreadPoolJob *job = (SgenThreadPoolJob *)pool->job_queue.data [i];
if (job->state == STATE_WAITING) {
job->state = STATE_IN_PROGRESS;
return job;
/* Assumes that the lock is held. */
static ssize_t
-find_job_in_queue (SgenThreadPoolJob *job)
+find_job_in_queue (SgenThreadPool *pool, SgenThreadPoolJob *job)
{
- for (ssize_t i = 0; i < job_queue.next_slot; ++i) {
- if (job_queue.data [i] == job)
+ for (ssize_t i = 0; i < pool->job_queue.next_slot; ++i) {
+ if (pool->job_queue.data [i] == job)
return i;
}
return -1;
/* Assumes that the lock is held. */
static void
-remove_job (SgenThreadPoolJob *job)
+remove_job (SgenThreadPool *pool, SgenThreadPoolJob *job)
{
ssize_t index;
SGEN_ASSERT (0, job->state == STATE_DONE, "Why are we removing a job that's not done?");
- index = find_job_in_queue (job);
+ index = find_job_in_queue (pool, job);
SGEN_ASSERT (0, index >= 0, "Why is the job we're trying to remove not in the queue?");
- job_queue.data [index] = NULL;
- sgen_pointer_queue_remove_nulls (&job_queue);
+ pool->job_queue.data [index] = NULL;
+ sgen_pointer_queue_remove_nulls (&pool->job_queue);
sgen_thread_pool_job_free (job);
}
static gboolean
-continue_idle_job (void *thread_data)
+continue_idle_job (SgenThreadPool *pool, void *thread_data)
{
- if (!continue_idle_job_func)
+ if (!pool->continue_idle_job_func)
return FALSE;
- return continue_idle_job_func (thread_data);
+ return pool->continue_idle_job_func (thread_data);
}
static gboolean
-should_work (void *thread_data)
+should_work (SgenThreadPool *pool, void *thread_data)
{
- if (!should_work_func)
+ if (!pool->should_work_func)
return TRUE;
- return should_work_func (thread_data);
+ return pool->should_work_func (thread_data);
}
static mono_native_thread_return_t
-thread_func (void *thread_data)
+thread_func (SgenThreadPoolData *thread_data)
{
- thread_init_func (thread_data);
+ SgenThreadPool *pool = thread_data->pool;
+
+ pool->thread_init_func (thread_data);
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
for (;;) {
gboolean do_idle;
SgenThreadPoolJob *job;
- if (!should_work (thread_data)) {
- mono_os_cond_wait (&work_cond, &lock);
+ if (!should_work (pool, thread_data) && !pool->threadpool_shutdown) {
+ mono_os_cond_wait (&pool->work_cond, &pool->lock);
continue;
}
/*
* main thread might then set continue idle and signal us before we can take
* the lock, and we'd lose the signal.
*/
- do_idle = continue_idle_job (thread_data);
- job = get_job_and_set_in_progress ();
+ do_idle = continue_idle_job (pool, thread_data);
+ job = get_job_and_set_in_progress (pool);
- if (!job && !do_idle && !threadpool_shutdown) {
+ if (!job && !do_idle && !pool->threadpool_shutdown) {
/*
* pthread_cond_wait() can return successfully despite the condition
* not being signalled, so we have to run this in a loop until we
* really have work to do.
*/
- mono_os_cond_wait (&work_cond, &lock);
+ mono_os_cond_wait (&pool->work_cond, &pool->lock);
continue;
}
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_unlock (&pool->lock);
if (job) {
job->func (thread_data, job);
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
SGEN_ASSERT (0, job->state == STATE_IN_PROGRESS, "The job should still be in progress.");
job->state = STATE_DONE;
- remove_job (job);
+ remove_job (pool, job);
/*
* Only the main GC thread will ever wait on the done condition, so we don't
* have to broadcast.
*/
- mono_os_cond_signal (&done_cond);
+ mono_os_cond_signal (&pool->done_cond);
} else if (do_idle) {
- SGEN_ASSERT (0, idle_job_func, "Why do we have idle work when there's no idle job function?");
+ SGEN_ASSERT (0, pool->idle_job_func, "Why do we have idle work when there's no idle job function?");
do {
- idle_job_func (thread_data);
- do_idle = continue_idle_job (thread_data);
- } while (do_idle && !job_queue.next_slot);
+ pool->idle_job_func (thread_data);
+ do_idle = continue_idle_job (pool, thread_data);
+ } while (do_idle && !pool->job_queue.next_slot);
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
if (!do_idle)
- mono_os_cond_signal (&done_cond);
+ mono_os_cond_signal (&pool->done_cond);
} else {
- SGEN_ASSERT (0, threadpool_shutdown, "Why did we unlock if no jobs and not shutting down?");
- mono_os_mutex_lock (&lock);
- threads_finished++;
- mono_os_cond_signal (&done_cond);
- mono_os_mutex_unlock (&lock);
+ SGEN_ASSERT (0, pool->threadpool_shutdown, "Why did we unlock if no jobs and not shutting down?");
+ mono_os_mutex_lock (&pool->lock);
+ pool->threads_finished++;
+ mono_os_cond_signal (&pool->done_cond);
+ mono_os_mutex_unlock (&pool->lock);
return 0;
}
}
}
void
-sgen_thread_pool_init (int num_threads, SgenThreadPoolThreadInitFunc init_func, SgenThreadPoolIdleJobFunc idle_func, SgenThreadPoolContinueIdleJobFunc continue_idle_func, SgenThreadPoolShouldWorkFunc should_work_func_p, void **thread_datas)
+sgen_thread_pool_init (SgenThreadPool *pool, int num_threads, SgenThreadPoolThreadInitFunc init_func, SgenThreadPoolIdleJobFunc idle_func, SgenThreadPoolContinueIdleJobFunc continue_idle_func, SgenThreadPoolShouldWorkFunc should_work_func_p, SgenThreadPoolData **thread_datas)
{
int i;
- threads_num = (num_threads < MAX_NUM_THREADS) ? num_threads : MAX_NUM_THREADS;
+ SGEN_ASSERT (0, num_threads > 0, "Why are we creating a threadpool with no threads?");
+
+ pool->threads_num = (num_threads < MAX_NUM_THREADS) ? num_threads : MAX_NUM_THREADS;
- mono_os_mutex_init (&lock);
- mono_os_cond_init (&work_cond);
- mono_os_cond_init (&done_cond);
+ mono_os_mutex_init (&pool->lock);
+ mono_os_cond_init (&pool->work_cond);
+ mono_os_cond_init (&pool->done_cond);
- thread_init_func = init_func;
- idle_job_func = idle_func;
- continue_idle_job_func = continue_idle_func;
- should_work_func = should_work_func_p;
+ pool->thread_init_func = init_func;
+ pool->idle_job_func = idle_func;
+ pool->continue_idle_job_func = continue_idle_func;
+ pool->should_work_func = should_work_func_p;
- for (i = 0; i < threads_num; i++)
- mono_native_thread_create (&threads [i], thread_func, thread_datas ? thread_datas [i] : NULL);
+ sgen_pointer_queue_init (&pool->job_queue, 0);
+ pool->threads_finished = 0;
+ pool->threadpool_shutdown = FALSE;
+
+ for (i = 0; i < pool->threads_num; i++) {
+ thread_datas [i]->pool = pool;
+ mono_native_thread_create (&pool->threads [i], thread_func, thread_datas [i]);
+ }
}
void
-sgen_thread_pool_shutdown (void)
+sgen_thread_pool_shutdown (SgenThreadPool *pool)
{
- if (!threads_num)
+ if (!pool)
return;
- mono_os_mutex_lock (&lock);
- threadpool_shutdown = TRUE;
- mono_os_cond_broadcast (&work_cond);
- while (threads_finished < threads_num)
- mono_os_cond_wait (&done_cond, &lock);
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_lock (&pool->lock);
+ pool->threadpool_shutdown = TRUE;
+ mono_os_cond_broadcast (&pool->work_cond);
+ while (pool->threads_finished < pool->threads_num)
+ mono_os_cond_wait (&pool->done_cond, &pool->lock);
+ mono_os_mutex_unlock (&pool->lock);
- mono_os_mutex_destroy (&lock);
- mono_os_cond_destroy (&work_cond);
- mono_os_cond_destroy (&done_cond);
+ mono_os_mutex_destroy (&pool->lock);
+ mono_os_cond_destroy (&pool->work_cond);
+ mono_os_cond_destroy (&pool->done_cond);
}
SgenThreadPoolJob*
}
void
-sgen_thread_pool_job_enqueue (SgenThreadPoolJob *job)
+sgen_thread_pool_job_enqueue (SgenThreadPool *pool, SgenThreadPoolJob *job)
{
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
- sgen_pointer_queue_add (&job_queue, job);
- mono_os_cond_signal (&work_cond);
+ sgen_pointer_queue_add (&pool->job_queue, job);
+ mono_os_cond_signal (&pool->work_cond);
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_unlock (&pool->lock);
}
void
-sgen_thread_pool_job_wait (SgenThreadPoolJob *job)
+sgen_thread_pool_job_wait (SgenThreadPool *pool, SgenThreadPoolJob *job)
{
SGEN_ASSERT (0, job, "Where's the job?");
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
- while (find_job_in_queue (job) >= 0)
- mono_os_cond_wait (&done_cond, &lock);
+ while (find_job_in_queue (pool, job) >= 0)
+ mono_os_cond_wait (&pool->done_cond, &pool->lock);
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_unlock (&pool->lock);
}
void
-sgen_thread_pool_idle_signal (void)
+sgen_thread_pool_idle_signal (SgenThreadPool *pool)
{
- SGEN_ASSERT (0, idle_job_func, "Why are we signaling idle without an idle function?");
+ SGEN_ASSERT (0, pool->idle_job_func, "Why are we signaling idle without an idle function?");
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
- if (continue_idle_job_func (NULL))
- mono_os_cond_broadcast (&work_cond);
+ if (pool->continue_idle_job_func (NULL))
+ mono_os_cond_broadcast (&pool->work_cond);
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_unlock (&pool->lock);
}
void
-sgen_thread_pool_idle_wait (void)
+sgen_thread_pool_idle_wait (SgenThreadPool *pool)
{
- SGEN_ASSERT (0, idle_job_func, "Why are we waiting for idle without an idle function?");
+ SGEN_ASSERT (0, pool->idle_job_func, "Why are we waiting for idle without an idle function?");
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
- while (continue_idle_job_func (NULL))
- mono_os_cond_wait (&done_cond, &lock);
+ while (pool->continue_idle_job_func (NULL))
+ mono_os_cond_wait (&pool->done_cond, &pool->lock);
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_unlock (&pool->lock);
}
void
-sgen_thread_pool_wait_for_all_jobs (void)
+sgen_thread_pool_wait_for_all_jobs (SgenThreadPool *pool)
{
- mono_os_mutex_lock (&lock);
+ mono_os_mutex_lock (&pool->lock);
- while (!sgen_pointer_queue_is_empty (&job_queue))
- mono_os_cond_wait (&done_cond, &lock);
+ while (!sgen_pointer_queue_is_empty (&pool->job_queue))
+ mono_os_cond_wait (&pool->done_cond, &pool->lock);
- mono_os_mutex_unlock (&lock);
+ mono_os_mutex_unlock (&pool->lock);
}
/* Return 0 if is not a thread pool thread or the thread number otherwise */
int
-sgen_thread_pool_is_thread_pool_thread (MonoNativeThreadId some_thread)
+sgen_thread_pool_is_thread_pool_thread (SgenThreadPool *pool, MonoNativeThreadId some_thread)
{
int i;
- for (i = 0; i < threads_num; i++) {
- if (some_thread == threads [i])
+ if (!pool)
+ return 0;
+
+ for (i = 0; i < pool->threads_num; i++) {
+ if (some_thread == pool->threads [i])
return i + 1;
}
#ifndef __MONO_SGEN_THREAD_POOL_H__
#define __MONO_SGEN_THREAD_POOL_H__
+#include "mono/sgen/sgen-pointer-queue.h"
+#include "mono/utils/mono-threads.h"
+
typedef struct _SgenThreadPoolJob SgenThreadPoolJob;
+typedef struct _SgenThreadPool SgenThreadPool;
+typedef struct _SgenThreadPoolData SgenThreadPoolData;
typedef void (*SgenThreadPoolJobFunc) (void *thread_data, SgenThreadPoolJob *job);
+typedef void (*SgenThreadPoolThreadInitFunc) (void*);
+typedef void (*SgenThreadPoolIdleJobFunc) (void*);
+typedef gboolean (*SgenThreadPoolContinueIdleJobFunc) (void*);
+typedef gboolean (*SgenThreadPoolShouldWorkFunc) (void*);
struct _SgenThreadPoolJob {
const char *name;
volatile gint32 state;
};
-typedef void (*SgenThreadPoolThreadInitFunc) (void*);
-typedef void (*SgenThreadPoolIdleJobFunc) (void*);
-typedef gboolean (*SgenThreadPoolContinueIdleJobFunc) (void*);
-typedef gboolean (*SgenThreadPoolShouldWorkFunc) (void*);
+#define MAX_NUM_THREADS 8
+
+struct _SgenThreadPool {
+ mono_mutex_t lock;
+ mono_cond_t work_cond;
+ mono_cond_t done_cond;
+
+ int threads_num;
+ MonoNativeThreadId threads [MAX_NUM_THREADS];
+
+ /* Only accessed with the lock held. */
+ SgenPointerQueue job_queue;
+
+ SgenThreadPoolThreadInitFunc thread_init_func;
+ SgenThreadPoolIdleJobFunc idle_job_func;
+ SgenThreadPoolContinueIdleJobFunc continue_idle_job_func;
+ SgenThreadPoolShouldWorkFunc should_work_func;
+
+ volatile gboolean threadpool_shutdown;
+ volatile int threads_finished;
+};
+
+struct _SgenThreadPoolData {
+ SgenThreadPool *pool;
+};
-void sgen_thread_pool_init (int num_threads, SgenThreadPoolThreadInitFunc init_func, SgenThreadPoolIdleJobFunc idle_func, SgenThreadPoolContinueIdleJobFunc continue_idle_func, SgenThreadPoolShouldWorkFunc should_work_func, void **thread_datas);
+void sgen_thread_pool_init (SgenThreadPool *pool, int num_threads, SgenThreadPoolThreadInitFunc init_func, SgenThreadPoolIdleJobFunc idle_func, SgenThreadPoolContinueIdleJobFunc continue_idle_func, SgenThreadPoolShouldWorkFunc should_work_func, SgenThreadPoolData **thread_datas);
-void sgen_thread_pool_shutdown (void);
+void sgen_thread_pool_shutdown (SgenThreadPool *pool);
SgenThreadPoolJob* sgen_thread_pool_job_alloc (const char *name, SgenThreadPoolJobFunc func, size_t size);
/* This only needs to be called on jobs that are not enqueued. */
void sgen_thread_pool_job_free (SgenThreadPoolJob *job);
-void sgen_thread_pool_job_enqueue (SgenThreadPoolJob *job);
+void sgen_thread_pool_job_enqueue (SgenThreadPool *pool, SgenThreadPoolJob *job);
/* This must only be called after the job has been enqueued. */
-void sgen_thread_pool_job_wait (SgenThreadPoolJob *job);
+void sgen_thread_pool_job_wait (SgenThreadPool *pool, SgenThreadPoolJob *job);
-void sgen_thread_pool_idle_signal (void);
-void sgen_thread_pool_idle_wait (void);
+void sgen_thread_pool_idle_signal (SgenThreadPool *pool);
+void sgen_thread_pool_idle_wait (SgenThreadPool *pool);
-void sgen_thread_pool_wait_for_all_jobs (void);
+void sgen_thread_pool_wait_for_all_jobs (SgenThreadPool *pool);
-int sgen_thread_pool_is_thread_pool_thread (MonoNativeThreadId thread);
+int sgen_thread_pool_is_thread_pool_thread (SgenThreadPool *pool, MonoNativeThreadId thread);
#endif
static WorkerData *workers_data;
static SgenWorkerCallback worker_init_cb;
+static SgenThreadPool pool_inst;
+static SgenThreadPool *pool; /* null if we're not using workers */
+
/*
* When using multiple workers, we need to have the last worker
* enqueue the preclean jobs (if there are any). This lock ensures
STATE_WORK_ENQUEUED
};
+#define SGEN_WORKER_MIN_SECTIONS_SIGNAL 4
+
typedef gint32 State;
static SgenObjectOperations * volatile idle_func_object_ops;
else if (new_state == STATE_WORKING)
SGEN_ASSERT (0, old_state == STATE_WORK_ENQUEUED, "We can only transition to WORKING from WORK ENQUEUED");
if (new_state == STATE_NOT_WORKING || new_state == STATE_WORKING)
- SGEN_ASSERT (6, sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "Only the worker thread is allowed to transition to NOT_WORKING or WORKING");
+ SGEN_ASSERT (6, sgen_thread_pool_is_thread_pool_thread (pool, mono_native_thread_id_get ()), "Only the worker thread is allowed to transition to NOT_WORKING or WORKING");
return InterlockedCompareExchange (&data->state, new_state, old_state) == old_state;
}
}
if (need_signal)
- sgen_thread_pool_idle_signal ();
+ sgen_thread_pool_idle_signal (pool);
}
static void
return;
}
- sgen_thread_pool_job_enqueue (job);
+ sgen_thread_pool_job_enqueue (pool, job);
}
static gboolean
workers_get_work (WorkerData *data)
{
- SgenMajorCollector *major;
+ SgenMajorCollector *major = sgen_get_major_collector ();
+ SgenMinorCollector *minor = sgen_get_minor_collector ();
+ GrayQueueSection *section;
g_assert (sgen_gray_object_queue_is_empty (&data->private_gray_queue));
+ g_assert (major->is_concurrent || minor->is_parallel);
- /* If we're concurrent, steal from the workers distribute gray queue. */
- major = sgen_get_major_collector ();
- if (major->is_concurrent) {
- GrayQueueSection *section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
- if (section) {
- sgen_gray_object_enqueue_section (&data->private_gray_queue, section, major->is_parallel);
- return TRUE;
- }
+ section = sgen_section_gray_queue_dequeue (&workers_distribute_gray_queue);
+ if (section) {
+ sgen_gray_object_enqueue_section (&data->private_gray_queue, section, major->is_parallel);
+ return TRUE;
}
/* Nobody to steal from */
workers_steal_work (WorkerData *data)
{
SgenMajorCollector *major = sgen_get_major_collector ();
+ SgenMinorCollector *minor = sgen_get_minor_collector ();
+ int generation = sgen_get_current_collection_generation ();
GrayQueueSection *section = NULL;
int i, current_worker;
- if (!major->is_parallel)
+ if ((generation == GENERATION_OLD && !major->is_parallel) ||
+ (generation == GENERATION_NURSERY && !minor->is_parallel))
return FALSE;
/* If we're parallel, steal from other workers' private gray queues */
{
WorkerData *data = (WorkerData *)data_untyped;
SgenMajorCollector *major = sgen_get_major_collector ();
+ SgenMinorCollector *minor = sgen_get_minor_collector ();
sgen_client_thread_register_worker ();
- if (!major->is_concurrent)
+ if (!major->is_concurrent && !minor->is_parallel)
return;
init_private_gray_queue (data);
WorkerData *data = (WorkerData *)data_untyped;
SGEN_ASSERT (0, continue_idle_func (data_untyped), "Why are we called when we're not supposed to work?");
- SGEN_ASSERT (0, sgen_concurrent_collection_in_progress (), "The worker should only mark in concurrent collections.");
if (data->state == STATE_WORK_ENQUEUED) {
set_state (data, STATE_WORK_ENQUEUED, STATE_WORKING);
sgen_drain_gray_stack (ctx);
- if (data->private_gray_queue.num_sections > 16 && workers_finished && worker_awakenings < active_workers_num) {
+ if (data->private_gray_queue.num_sections >= SGEN_WORKER_MIN_SECTIONS_SIGNAL
+ && workers_finished && worker_awakenings < active_workers_num) {
/* We bound the number of worker awakenings just to be sure */
worker_awakenings++;
mono_os_mutex_lock (&finished_lock);
void
sgen_workers_init_distribute_gray_queue (void)
{
- SGEN_ASSERT (0, sgen_get_major_collector ()->is_concurrent,
+ SGEN_ASSERT (0, sgen_get_major_collector ()->is_concurrent || sgen_get_minor_collector ()->is_parallel,
"Why should we init the distribute gray queue if we don't need it?");
init_distribute_gray_queue ();
}
sgen_workers_init (int num_workers, SgenWorkerCallback callback)
{
int i;
- void **workers_data_ptrs = (void **)alloca(num_workers * sizeof(void *));
-
- if (!sgen_get_major_collector ()->is_concurrent) {
- sgen_thread_pool_init (num_workers, thread_pool_init_func, NULL, NULL, NULL, NULL);
- return;
- }
+ WorkerData **workers_data_ptrs = (WorkerData**)alloca(num_workers * sizeof(WorkerData*));
mono_os_mutex_init (&finished_lock);
//g_print ("initing %d workers\n", num_workers);
init_distribute_gray_queue ();
for (i = 0; i < num_workers; ++i)
- workers_data_ptrs [i] = (void *) &workers_data [i];
+ workers_data_ptrs [i] = &workers_data [i];
worker_init_cb = callback;
- sgen_thread_pool_init (num_workers, thread_pool_init_func, marker_idle_func, continue_idle_func, should_work_func, workers_data_ptrs);
+ pool = &pool_inst;
+ sgen_thread_pool_init (pool, num_workers, thread_pool_init_func, marker_idle_func, continue_idle_func, should_work_func, (SgenThreadPoolData**)workers_data_ptrs);
mono_counters_register ("# workers finished", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_workers_num_finished);
}
+void
+sgen_workers_shutdown (void)
+{
+ if (pool)
+ sgen_thread_pool_shutdown (pool);
+}
+
void
sgen_workers_stop_all_workers (void)
{
mono_memory_write_barrier ();
forced_stop = TRUE;
- sgen_thread_pool_wait_for_all_jobs ();
- sgen_thread_pool_idle_wait ();
+ sgen_thread_pool_wait_for_all_jobs (pool);
+ sgen_thread_pool_idle_wait (pool);
SGEN_ASSERT (0, sgen_workers_all_done (), "Can only signal enqueue work when in no work state");
}
{
int i;
- sgen_thread_pool_wait_for_all_jobs ();
- sgen_thread_pool_idle_wait ();
+ sgen_thread_pool_wait_for_all_jobs (pool);
+ sgen_thread_pool_idle_wait (pool);
SGEN_ASSERT (0, sgen_workers_all_done (), "Can only signal enqueue work when in no work state");
/* At this point all the workers have stopped. */
callback (&workers_data [i]);
}
+gboolean
+sgen_workers_is_worker_thread (MonoNativeThreadId id)
+{
+ if (!pool)
+ return FALSE;
+ return sgen_thread_pool_is_thread_pool_thread (pool, id);
+}
+
#endif
typedef struct _WorkerData WorkerData;
struct _WorkerData {
+ /*
+ * Threadpool threads receive as their starting argument a WorkerData.
+ * tp_data is meant for use inside the sgen thread pool and must be first.
+ */
+ SgenThreadPoolData tp_data;
gint32 state;
SgenGrayQueue private_gray_queue; /* only read/written by worker thread */
/*
typedef void (*SgenWorkerCallback) (WorkerData *data);
void sgen_workers_init (int num_workers, SgenWorkerCallback callback);
+void sgen_workers_shutdown (void);
void sgen_workers_stop_all_workers (void);
void sgen_workers_set_num_active_workers (int num_workers);
void sgen_workers_start_all_workers (SgenObjectOperations *object_ops_nopar, SgenObjectOperations *object_ops_par, SgenWorkersFinishCallback finish_job);
SgenObjectOperations* sgen_workers_get_idle_func_object_ops (void);
int sgen_workers_get_job_split_count (void);
void sgen_workers_foreach (SgenWorkerCallback callback);
+gboolean sgen_workers_is_worker_thread (MonoNativeThreadId id);
#endif
projects / mono.git / commitdiff