*
* Author:
* Paolo Molaro (lupus@ximian.com)
+ * Rodrigo Kumpera (kumpera@gmail.com)
*
* Copyright 2005-2010 Novell, Inc (http://www.novell.com)
*
*
* Copyright 2001-2003 Ximian, Inc
* Copyright 2003-2010 Novell, Inc.
+ * Copyright 2011 Xamarin, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
A good place to start is add_nursery_frag. The tricky thing here is
placing those objects atomically outside of a collection.
-
+ *) Allocation should use asymmetric Dekker synchronization:
+ http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
+ This should help weak consistency archs.
*/
#include "config.h"
#ifdef HAVE_SGEN_GC
#include "utils/mono-semaphore.h"
#include "utils/mono-counters.h"
#include "utils/mono-proclib.h"
+#include "utils/mono-memory-model.h"
#include <mono/utils/memcheck.h>
/* If set, do a plausibility check on the scan_starts before and after
each collection */
static gboolean do_scan_starts_check = FALSE;
+static gboolean nursery_collection_is_parallel = FALSE;
static gboolean disable_minor_collections = FALSE;
static gboolean disable_major_collections = FALSE;
int gc_debug_level = 0;
FILE* gc_debug_file;
+static gboolean debug_print_allowance = FALSE;
/*
void
#endif
#ifndef DISABLE_CRITICAL_REGION
-/* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
-#define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
-#define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
+
+/* Enter must be visible before anything is done in the critical region. */
+#define ENTER_CRITICAL_REGION do { mono_atomic_store_release (&IN_CRITICAL_REGION, 1); } while (0)
+
+/* Exit must make sure all critical regions stores are visible before it signal the end of the region.
+ * We don't need to emit a full barrier since we
+ */
+#define EXIT_CRITICAL_REGION do { mono_atomic_store_seq (&IN_CRITICAL_REGION, 0); } while (0)
+
+
#endif
/*
*/
/*heap limits*/
static mword max_heap_size = ((mword)0)- ((mword)1);
+static mword soft_heap_limit = ((mword)0) - ((mword)1);
static mword allocated_heap;
/*Object was pinned during the current collection*/
}
static void
-init_heap_size_limits (glong max_heap)
+init_heap_size_limits (glong max_heap, glong soft_limit)
{
+ if (soft_limit)
+ soft_heap_limit = soft_limit;
+
if (max_heap == 0)
return;
+ if (max_heap < soft_limit) {
+ fprintf (stderr, "max-heap-size must be at least as large as soft-heap-limit.\n");
+ exit (1);
+ }
+
if (max_heap < nursery_size * 4) {
fprintf (stderr, "max-heap-size must be at least 4 times larger than nursery size.\n");
exit (1);
typedef SgenGrayQueue GrayQueue;
-typedef void (*CopyOrMarkObjectFunc) (void**, GrayQueue*);
-typedef char* (*ScanObjectFunc) (char*, GrayQueue*);
-
/* forward declarations */
static int stop_world (int generation);
static int restart_world (int generation);
static gboolean need_major_collection (mword space_needed);
static void major_collection (const char *reason);
+static gboolean collection_is_parallel (void);
+
static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track, gboolean in_gc);
static gboolean mono_gc_is_critical_method (MonoMethod *method);
return bitmap;
}
+ case DESC_TYPE_LARGE_BITMAP: {
+ gsize bmap = (d >> LOW_TYPE_BITS) << OBJECT_HEADER_WORDS;
+
+ bitmap = g_new0 (gsize, 1);
+ bitmap [0] = bmap;
+ *numbits = 0;
+ while (bmap) {
+ (*numbits) ++;
+ bmap >>= 1;
+ }
+ return bitmap;
+ }
default:
g_assert_not_reached ();
}
mono_sgen_add_to_global_remset (gpointer ptr)
{
RememberedSet *rs;
- gboolean lock = major_collector.is_parallel;
+ gboolean lock = collection_is_parallel ();
if (use_cardtable) {
sgen_card_table_mark_address ((mword)ptr);
char *obj;
if (current_collection_generation == GENERATION_NURSERY) {
+ ScanObjectFunc scan_func = mono_sgen_get_minor_scan_object ();
+
for (;;) {
GRAY_OBJECT_DEQUEUE (queue, obj);
if (!obj)
return TRUE;
DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
- major_collector.minor_scan_object (obj, queue);
+ scan_func (obj, queue);
}
} else {
int i;
- if (major_collector.is_parallel && queue == &workers_distribute_gray_queue)
+ if (collection_is_parallel () && queue == &workers_distribute_gray_queue)
return TRUE;
do {
void
mono_sgen_pin_object (void *object, GrayQueue *queue)
{
- if (major_collector.is_parallel) {
+ if (collection_is_parallel ()) {
LOCK_PIN_QUEUE;
/*object arrives pinned*/
pin_stage_ptr (object);
} while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
}
+static unsigned long
+prot_flags_for_activate (int activate)
+{
+ unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
+ return prot_flags | MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
+}
+
+/*
+ * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
+ * This must not require any lock.
+ */
+void*
+mono_sgen_alloc_os_memory (size_t size, int activate)
+{
+ void *ptr = mono_valloc (0, size, prot_flags_for_activate (activate));
+ if (ptr) {
+ /* FIXME: CAS */
+ total_alloc += size;
+ }
+ return ptr;
+}
+
/* size must be a power of 2 */
void*
mono_sgen_alloc_os_memory_aligned (mword size, mword alignment, gboolean activate)
{
- /* Allocate twice the memory to be able to put the block on an aligned address */
- char *mem = mono_sgen_alloc_os_memory (size + alignment, activate);
- char *aligned;
-
- g_assert (mem);
-
- aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
- g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
-
- if (aligned > mem)
- mono_sgen_free_os_memory (mem, aligned - mem);
- if (aligned + size < mem + size + alignment)
- mono_sgen_free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
+ void *ptr = mono_valloc_aligned (size, alignment, prot_flags_for_activate (activate));
+ if (ptr) {
+ /* FIXME: CAS */
+ total_alloc += size;
+ }
+ return ptr;
+}
- return aligned;
+/*
+ * Free the memory returned by mono_sgen_alloc_os_memory (), returning it to the OS.
+ */
+void
+mono_sgen_free_os_memory (void *addr, size_t size)
+{
+ mono_vfree (addr, size);
+ /* FIXME: CAS */
+ total_alloc -= size;
}
/*
finalized_array [finalized_array_entries++] = object;
}
+static void
+bridge_process (void)
+{
+ if (finalized_array_entries <= 0)
+ return;
+
+ g_assert (mono_sgen_need_bridge_processing ());
+ mono_sgen_bridge_processing (finalized_array_entries, finalized_array);
+
+ finalized_array_entries = 0;
+}
+
+CopyOrMarkObjectFunc
+mono_sgen_get_copy_object (void)
+{
+ if (current_collection_generation == GENERATION_NURSERY) {
+ if (collection_is_parallel ())
+ return major_collector.copy_object;
+ else
+ return major_collector.nopar_copy_object;
+ } else {
+ return major_collector.copy_or_mark_object;
+ }
+}
+
+ScanObjectFunc
+mono_sgen_get_minor_scan_object (void)
+{
+ g_assert (current_collection_generation == GENERATION_NURSERY);
+
+ if (collection_is_parallel ())
+ return major_collector.minor_scan_object;
+ else
+ return major_collector.nopar_minor_scan_object;
+}
+
+ScanVTypeFunc
+mono_sgen_get_minor_scan_vtype (void)
+{
+ g_assert (current_collection_generation == GENERATION_NURSERY);
+
+ if (collection_is_parallel ())
+ return major_collector.minor_scan_vtype;
+ else
+ return major_collector.nopar_minor_scan_vtype;
+}
+
static void
finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
{
int fin_ready;
int ephemeron_rounds = 0;
int num_loops;
- CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? major_collector.copy_object : major_collector.copy_or_mark_object;
+ CopyOrMarkObjectFunc copy_func = mono_sgen_get_copy_object ();
/*
* We copied all the reachable objects. Now it's the time to copy
if (generation == GENERATION_OLD)
finalize_in_range (copy_func, nursery_start, nursery_end, GENERATION_NURSERY, queue);
- if (fin_ready != num_ready_finalizers) {
+ if (fin_ready != num_ready_finalizers)
++num_loops;
- if (finalized_array != NULL)
- mono_sgen_bridge_processing (finalized_array_entries, finalized_array);
- }
/* drain the new stack that might have been created */
DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
/* FIXME: handle this for parallel collector */
- g_assert (!major_collector.is_parallel);
+ g_assert (!collection_is_parallel ());
if (moved_objects_idx == MOVED_OBJECTS_NUM) {
mono_profiler_gc_moves (moved_objects, moved_objects_idx);
try_calculate_minor_collection_allowance (gboolean overwrite)
{
int num_major_sections, num_major_sections_saved, save_target, allowance_target;
- mword los_memory_saved;
+ mword los_memory_saved, new_major, new_heap_size;
if (overwrite)
g_assert (need_calculate_minor_collection_allowance);
num_major_sections_saved = MAX (last_collection_old_num_major_sections - num_major_sections, 0);
los_memory_saved = MAX (last_collection_old_los_memory_usage - last_collection_los_memory_usage, 1);
- save_target = ((num_major_sections * major_collector.section_size) + los_memory_saved) / 2;
+ new_major = num_major_sections * major_collector.section_size;
+ new_heap_size = new_major + last_collection_los_memory_usage;
+
+ /*
+ * FIXME: Why is save_target half the major memory plus half the
+ * LOS memory saved? Shouldn't it be half the major memory
+ * saved plus half the LOS memory saved? Or half the whole heap
+ * size?
+ */
+ save_target = (new_major + los_memory_saved) / 2;
/*
* We aim to allow the allocation of as many sections as is
minor_collection_allowance = MAX (MIN (allowance_target, num_major_sections * major_collector.section_size + los_memory_usage), MIN_MINOR_COLLECTION_ALLOWANCE);
+ if (new_heap_size + minor_collection_allowance > soft_heap_limit) {
+ if (new_heap_size > soft_heap_limit)
+ minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
+ else
+ minor_collection_allowance = MAX (soft_heap_limit - new_heap_size, MIN_MINOR_COLLECTION_ALLOWANCE);
+ }
+
+ if (debug_print_allowance) {
+ mword old_major = last_collection_old_num_major_sections * major_collector.section_size;
+
+ fprintf (gc_debug_file, "Before collection: %ld bytes (%ld major, %ld LOS)\n",
+ old_major + last_collection_old_los_memory_usage, old_major, last_collection_old_los_memory_usage);
+ fprintf (gc_debug_file, "After collection: %ld bytes (%ld major, %ld LOS)\n",
+ new_heap_size, new_major, last_collection_los_memory_usage);
+ fprintf (gc_debug_file, "Allowance: %ld bytes\n", minor_collection_allowance);
+ }
+
if (major_collector.have_computed_minor_collection_allowance)
major_collector.have_computed_minor_collection_allowance ();
return need_major_collection (space_needed);
}
+static gboolean
+collection_is_parallel (void)
+{
+ switch (current_collection_generation) {
+ case GENERATION_NURSERY:
+ return nursery_collection_is_parallel;
+ case GENERATION_OLD:
+ return major_collector.is_parallel;
+ default:
+ g_assert_not_reached ();
+ }
+}
+
+gboolean
+mono_sgen_nursery_collection_is_parallel (void)
+{
+ return nursery_collection_is_parallel;
+}
+
static GrayQueue*
job_gray_queue (WorkerData *worker_data)
{
time_minor_scan_card_table += TV_ELAPSED_MS (atv, btv);
}
- if (!major_collector.is_parallel)
+ if (!collection_is_parallel ())
drain_gray_stack (&gray_queue, -1);
if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
/* registered roots, this includes static fields */
- scrrjd_normal.func = major_collector.copy_object;
+ scrrjd_normal.func = collection_is_parallel () ? major_collector.copy_object : major_collector.nopar_copy_object;
scrrjd_normal.heap_start = nursery_start;
scrrjd_normal.heap_end = nursery_next;
scrrjd_normal.root_type = ROOT_TYPE_NORMAL;
workers_enqueue_job (job_scan_from_registered_roots, &scrrjd_normal);
- scrrjd_wbarrier.func = major_collector.copy_object;
+ scrrjd_wbarrier.func = collection_is_parallel () ? major_collector.copy_object : major_collector.nopar_copy_object;
scrrjd_wbarrier.heap_start = nursery_start;
scrrjd_wbarrier.heap_end = nursery_next;
scrrjd_wbarrier.root_type = ROOT_TYPE_WBARRIER;
time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
btv = atv;
- if (major_collector.is_parallel) {
+ if (collection_is_parallel ()) {
while (!gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) {
workers_distribute_gray_queue_sections ();
- usleep (1000);
+ g_usleep (1000);
}
}
workers_join ();
- if (major_collector.is_parallel)
+ if (collection_is_parallel ())
g_assert (gray_object_queue_is_empty (&gray_queue));
finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY, &gray_queue);
if (major_collector.is_parallel) {
while (!gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) {
workers_distribute_gray_queue_sections ();
- usleep (1000);
+ g_usleep (1000);
}
}
workers_join ();
major_collector.report_pinned_memory_usage ();
}
-/*
- * Allocate a big chunk of memory from the OS (usually 64KB to several megabytes).
- * This must not require any lock.
- */
-void*
-mono_sgen_alloc_os_memory (size_t size, int activate)
-{
- void *ptr;
- unsigned long prot_flags = activate? MONO_MMAP_READ|MONO_MMAP_WRITE: MONO_MMAP_NONE;
-
- prot_flags |= MONO_MMAP_PRIVATE | MONO_MMAP_ANON;
- size += pagesize - 1;
- size &= ~(pagesize - 1);
- ptr = mono_valloc (0, size, prot_flags);
- /* FIXME: CAS */
- total_alloc += size;
- return ptr;
-}
-
-/*
- * Free the memory returned by mono_sgen_alloc_os_memory (), returning it to the OS.
- */
-void
-mono_sgen_free_os_memory (void *addr, size_t size)
-{
- mono_vfree (addr, size);
-
- size += pagesize - 1;
- size &= ~(pagesize - 1);
- /* FIXME: CAS */
- total_alloc -= size;
-}
-
/*
* ######################################################################
* ######## Object allocation
*/
static void*
-alloc_degraded (MonoVTable *vtable, size_t size)
-{
+alloc_degraded (MonoVTable *vtable, size_t size, gboolean for_mature)
+{
+ static int last_major_gc_warned = -1;
+ static int num_degraded = 0;
+
+ if (!for_mature) {
+ if (last_major_gc_warned < num_major_gcs) {
+ ++num_degraded;
+ if (num_degraded == 1 || num_degraded == 3)
+ fprintf (stderr, "Warning: Degraded allocation. Consider increasing nursery-size if the warning persists.\n");
+ else if (num_degraded == 10)
+ fprintf (stderr, "Warning: Repeated degraded allocation. Consider increasing nursery-size.\n");
+ last_major_gc_warned = num_major_gcs;
+ }
+ }
+
if (need_major_collection (0)) {
mono_profiler_gc_event (MONO_GC_EVENT_START, 1);
stop_world (1);
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p , vtable, size);
g_assert (*p == NULL);
- *p = vtable;
-
- g_assert (TLAB_NEXT == new_next);
+ mono_atomic_store_seq (p, vtable);
return p;
}
* This avoids taking again the GC lock when registering, but this is moot when
* doing thread-local allocation, so it may not be a good idea.
*/
- g_assert (TLAB_NEXT == new_next);
if (TLAB_NEXT >= TLAB_REAL_END) {
+ int available_in_tlab;
/*
* Run out of space in the TLAB. When this happens, some amount of space
* remains in the TLAB, but not enough to satisfy the current allocation
* for a while, to decrease the number of useless nursery collections.
*/
if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE) {
- p = alloc_degraded (vtable, size);
+ p = alloc_degraded (vtable, size, FALSE);
binary_protocol_alloc_degraded (p, vtable, size);
return p;
}
- if (size > tlab_size) {
+ available_in_tlab = TLAB_REAL_END - TLAB_NEXT;
+ if (size > tlab_size || available_in_tlab > SGEN_MAX_NURSERY_WASTE) {
/* Allocate directly from the nursery */
do {
p = mono_sgen_nursery_alloc (size);
if (!p) {
minor_collect_or_expand_inner (size);
if (degraded_mode) {
- p = alloc_degraded (vtable, size);
+ p = alloc_degraded (vtable, size, FALSE);
binary_protocol_alloc_degraded (p, vtable, size);
return p;
} else {
int alloc_size = 0;
if (TLAB_START)
DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
+ mono_sgen_nursery_retire_region (p, available_in_tlab);
do {
p = mono_sgen_nursery_alloc_range (tlab_size, size, &alloc_size);
if (!p) {
minor_collect_or_expand_inner (tlab_size);
if (degraded_mode) {
- p = alloc_degraded (vtable, size);
+ p = alloc_degraded (vtable, size, FALSE);
binary_protocol_alloc_degraded (p, vtable, size);
return p;
} else {
/* Allocate from the TLAB */
p = (void*)TLAB_NEXT;
TLAB_NEXT += size;
- g_assert (TLAB_NEXT <= TLAB_REAL_END);
nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
}
if (G_LIKELY (p)) {
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p, vtable, size);
- *p = vtable;
+ mono_atomic_store_seq (p, vtable);
}
return p;
if (!p)
return NULL;
+ /*FIXME we should use weak memory ops here. Should help specially on x86. */
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
memset (p, 0, size);
- g_assert (*p == NULL);
} else {
+ int available_in_tlab;
+ char *real_end;
/* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
p = (void**)TLAB_NEXT;
/* FIXME: handle overflow */
new_next = (char*)p + size;
- TLAB_NEXT = new_next;
- /*First case, we overflowed the tlab, get a new one*/
- if (G_UNLIKELY (new_next >= TLAB_REAL_END)) {
+ real_end = TLAB_REAL_END;
+ available_in_tlab = real_end - (char*)p;
+
+ if (G_LIKELY (new_next < real_end)) {
+ TLAB_NEXT = new_next;
+ } else if (available_in_tlab > SGEN_MAX_NURSERY_WASTE) {
+ /* Allocate directly from the nursery */
+ p = mono_sgen_nursery_alloc (size);
+ if (!p)
+ return NULL;
+
+ if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
+ memset (p, 0, size);
+ } else {
int alloc_size = 0;
+ mono_sgen_nursery_retire_region (p, available_in_tlab);
new_next = mono_sgen_nursery_alloc_range (tlab_size, size, &alloc_size);
p = (void**)new_next;
if (!p)
return NULL;
- g_assert (alloc_size >= size);
TLAB_START = (char*)new_next;
TLAB_NEXT = new_next + size;
TLAB_REAL_END = new_next + alloc_size;
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
memset (new_next, 0, alloc_size);
new_next += size;
- g_assert (*p == NULL);
}
-
/* Second case, we overflowed temp end */
if (G_UNLIKELY (new_next >= TLAB_TEMP_END)) {
nursery_section->scan_starts [((char*)p - (char*)nursery_section->data)/SCAN_START_SIZE] = (char*)p;
TLAB_TEMP_END = MIN (TLAB_REAL_END, TLAB_NEXT + SCAN_START_SIZE);
DEBUG (5, fprintf (gc_debug_file, "Expanding local alloc: %p-%p\n", TLAB_NEXT, TLAB_TEMP_END));
}
-
- g_assert (TLAB_NEXT == new_next);
}
- /*
- * FIXME: We might need a memory barrier here so the change to tlab_next is
- * visible before the vtable store.
- */
-
HEAVY_STAT (++stat_objects_alloced);
HEAVY_STAT (stat_bytes_alloced += size);
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc (p, vtable, size);
- g_assert (*p == NULL);
- *p = vtable;
+ g_assert (*p == NULL); /* FIXME disable this in non debug builds */
+
+ mono_atomic_store_seq (p, vtable);
return p;
}
ENTER_CRITICAL_REGION;
arr = mono_gc_try_alloc_obj_nolock (vtable, size);
if (arr) {
+ /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
arr->max_length = max_length;
EXIT_CRITICAL_REGION;
return arr;
ENTER_CRITICAL_REGION;
str = mono_gc_try_alloc_obj_nolock (vtable, size);
if (str) {
+ /*This doesn't require fencing since EXIT_CRITICAL_REGION already does it for us*/
str->length = len;
EXIT_CRITICAL_REGION;
return str;
if (G_LIKELY (p)) {
DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
binary_protocol_alloc_pinned (p, vtable, size);
- *p = vtable;
+ mono_atomic_store_seq (p, vtable);
}
UNLOCK_GC;
return p;
void **res;
size_t size = ALIGN_UP (vtable->klass->instance_size);
LOCK_GC;
- res = alloc_degraded (vtable, size);
- *res = vtable;
+ res = alloc_degraded (vtable, size, TRUE);
+ mono_atomic_store_seq (res, vtable);
UNLOCK_GC;
if (G_UNLIKELY (vtable->klass->has_finalize))
mono_object_register_finalizer ((MonoObject*)res);
*/
static void
-rehash_roots (gboolean pinned)
+rehash_roots (int root_type)
{
int i;
unsigned int hash;
RootRecord *entry, *next;
int new_size;
- new_size = g_spaced_primes_closest (num_roots_entries [pinned]);
+ new_size = g_spaced_primes_closest (num_roots_entries [root_type]);
new_hash = mono_sgen_alloc_internal_dynamic (new_size * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
- for (i = 0; i < roots_hash_size [pinned]; ++i) {
- for (entry = roots_hash [pinned][i]; entry; entry = next) {
+ for (i = 0; i < roots_hash_size [root_type]; ++i) {
+ for (entry = roots_hash [root_type][i]; entry; entry = next) {
hash = mono_aligned_addr_hash (entry->start_root) % new_size;
next = entry->next;
entry->next = new_hash [hash];
new_hash [hash] = entry;
}
}
- mono_sgen_free_internal_dynamic (roots_hash [pinned], roots_hash_size [pinned] * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
- roots_hash [pinned] = new_hash;
- roots_hash_size [pinned] = new_size;
+ mono_sgen_free_internal_dynamic (roots_hash [root_type], roots_hash_size [root_type] * sizeof (RootRecord*), INTERNAL_MEM_ROOTS_TABLE);
+ roots_hash [root_type] = new_hash;
+ roots_hash_size [root_type] = new_size;
}
static RootRecord*
* ######################################################################
*/
-#if USE_SIGNAL_BASED_START_STOP_WORLD
-
-static MonoSemType suspend_ack_semaphore;
-static MonoSemType *suspend_ack_semaphore_ptr;
-static unsigned int global_stop_count = 0;
-
-static sigset_t suspend_signal_mask;
+unsigned int mono_sgen_global_stop_count = 0;
#ifdef USE_MONO_CTX
static MonoContext cur_thread_ctx = {0};
gc_callbacks.thread_suspend_func (info->runtime_data, NULL);
}
+void
+mono_sgen_fill_thread_info_for_suspend (SgenThreadInfo *info)
+{
+#ifdef HAVE_KW_THREAD
+ /* update the remset info in the thread data structure */
+ info->remset = remembered_set;
+#endif
+}
+
/*
* Define this and use the "xdomain-checks" MONO_GC_DEBUG option to
* have cross-domain checks in the write barrier.
static gboolean
is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
-void
-mono_sgen_wait_for_suspend_ack (int count)
-{
-#if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
- /* mach thread_resume is synchronous so we dont need to wait for them */
-#else
- int i, result;
-
- for (i = 0; i < count; ++i) {
- while ((result = MONO_SEM_WAIT (suspend_ack_semaphore_ptr)) != 0) {
- if (errno != EINTR) {
- g_error ("sem_wait ()");
- }
- }
- }
-#endif
-}
-
static int
restart_threads_until_none_in_managed_allocator (void)
{
return num_threads_died;
}
-static void
-suspend_thread (SgenThreadInfo *info, void *context)
-{
- int stop_count;
-#ifdef USE_MONO_CTX
- MonoContext monoctx;
-#else
- gpointer regs [ARCH_NUM_REGS];
-#endif
- gpointer stack_start;
-
- g_assert (info->doing_handshake);
-
- info->stopped_domain = mono_domain_get ();
- info->stopped_ip = context ? (gpointer) ARCH_SIGCTX_IP (context) : NULL;
- stop_count = global_stop_count;
- /* duplicate signal */
- if (0 && info->stop_count == stop_count)
- return;
-#ifdef HAVE_KW_THREAD
- /* update the remset info in the thread data structure */
- info->remset = remembered_set;
-#endif
- stack_start = context ? (char*) ARCH_SIGCTX_SP (context) - REDZONE_SIZE : NULL;
- /* If stack_start is not within the limits, then don't set it
- in info and we will be restarted. */
- if (stack_start >= info->stack_start_limit && info->stack_start <= info->stack_end) {
- info->stack_start = stack_start;
-
-#ifdef USE_MONO_CTX
- if (context) {
- mono_sigctx_to_monoctx (context, &monoctx);
- info->monoctx = &monoctx;
- } else {
- info->monoctx = NULL;
- }
-#else
- if (context) {
- ARCH_COPY_SIGCTX_REGS (regs, context);
- info->stopped_regs = regs;
- } else {
- info->stopped_regs = NULL;
- }
-#endif
- } else {
- g_assert (!info->stack_start);
- }
-
- /* Notify the JIT */
- if (gc_callbacks.thread_suspend_func)
- gc_callbacks.thread_suspend_func (info->runtime_data, context);
-
- DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)mono_native_thread_id_get ()));
- /* notify the waiting thread */
- MONO_SEM_POST (suspend_ack_semaphore_ptr);
- info->stop_count = stop_count;
-
- /* wait until we receive the restart signal */
- do {
- info->signal = 0;
- sigsuspend (&suspend_signal_mask);
- } while (info->signal != restart_signal_num && info->doing_handshake);
-
- DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)mono_native_thread_id_get ()));
- /* notify the waiting thread */
- MONO_SEM_POST (suspend_ack_semaphore_ptr);
-}
-
-/* LOCKING: assumes the GC lock is held (by the stopping thread) */
-static void
-suspend_handler (int sig, siginfo_t *siginfo, void *context)
-{
- SgenThreadInfo *info;
- int old_errno = errno;
-
- info = mono_thread_info_current ();
-
- if (info) {
- suspend_thread (info, context);
- } else {
- /* This can happen while a thread is dying */
- g_print ("no thread info in suspend\n");
- }
-
- errno = old_errno;
-}
-
-static void
-restart_handler (int sig)
-{
- SgenThreadInfo *info;
- int old_errno = errno;
-
- info = mono_thread_info_current ();
-
- /*
- * If a thread is dying there might be no thread info. In
- * that case we rely on info->doing_handshake.
- */
- if (info) {
- info->signal = restart_signal_num;
- DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)mono_native_thread_id_get ()));
- }
-
- errno = old_errno;
-}
-
static void
acquire_gc_locks (void)
{
update_current_thread_stack (&count);
- global_stop_count++;
- DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", global_stop_count, mono_thread_info_current (), (gpointer)mono_native_thread_id_get ()));
+ mono_sgen_global_stop_count++;
+ DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", mono_sgen_global_stop_count, mono_thread_info_current (), (gpointer)mono_native_thread_id_get ()));
TV_GETTIME (stop_world_time);
- count = mono_sgen_thread_handshake (suspend_signal_num);
+ count = mono_sgen_thread_handshake (TRUE);
count -= restart_threads_until_none_in_managed_allocator ();
g_assert (count >= 0);
DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count));
release_gc_locks ();
- count = mono_sgen_thread_handshake (restart_signal_num);
+ count = mono_sgen_thread_handshake (FALSE);
TV_GETTIME (end_sw);
usec = TV_ELAPSED (stop_world_time, end_sw);
max_pause_usec = MAX (usec, max_pause_usec);
DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec));
mono_profiler_gc_event (MONO_GC_EVENT_POST_START_WORLD, generation);
+
+ bridge_process ();
+
return count;
}
-#endif /* USE_SIGNAL_BASED_START_STOP_WORLD */
-
int
mono_sgen_get_current_collection_generation (void)
{
{
UserCopyOrMarkData *data = pthread_getspecific (user_copy_or_mark_key);
- if (current_collection_generation == GENERATION_NURSERY)
- major_collector.copy_object (&obj, data->queue);
- else
+ if (current_collection_generation == GENERATION_NURSERY) {
+ if (collection_is_parallel ())
+ major_collector.copy_object (&obj, data->queue);
+ else
+ major_collector.nopar_copy_object (&obj, data->queue);
+ } else {
major_collector.copy_or_mark_object (&obj, data->queue);
+ }
return obj;
}
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
return p + 1;
- major_collector.minor_scan_object ((char*)ptr, queue);
+ mono_sgen_get_minor_scan_object () ((char*)ptr, queue);
return p + 1;
case REMSET_VTYPE: {
+ ScanVTypeFunc scan_vtype = mono_sgen_get_minor_scan_vtype ();
size_t skip_size;
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
count = p [2];
skip_size = p [3];
while (count-- > 0) {
- major_collector.minor_scan_vtype ((char*)ptr, desc, queue);
+ scan_vtype ((char*)ptr, desc, queue);
ptr = (void**)((char*)ptr + skip_size);
}
return p + 4;
thread_info = info;
#endif
+#if !defined(__MACH__)
info->stop_count = -1;
- info->skip = 0;
info->signal = 0;
+#endif
+ info->skip = 0;
info->doing_handshake = FALSE;
info->thread_is_dying = FALSE;
info->stack_start = NULL;
FIXME: I believe we could avoid this by using mono_thread_info_lookup when
mono_thread_info_current returns NULL. Or fix mono_thread_info_lookup to do so.
*/
-#if defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED
+#if (defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED) || !defined(HAVE_PTHREAD_KILL)
LOCK_GC;
#else
while (!TRYLOCK_GC) {
- if (p->doing_handshake)
- suspend_thread (p, NULL);
- else
- usleep (50);
+ if (!mono_sgen_park_current_thread_if_doing_handshake (p))
+ g_usleep (50);
}
#endif
- binary_protocol_thread_unregister ((gpointer)id);
+ binary_protocol_thread_unregister ((gpointer)mono_thread_info_get_tid (p));
DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)mono_thread_info_get_tid (p)));
#if defined(__MACH__)
int
mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
{
- return mono_threads_pthread_create (new_thread, attr, start_routine, arg);
+ return pthread_create (new_thread, attr, start_routine, arg);
}
int
return;
}
DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr));
+ if (value)
+ binary_protocol_wbarrier (field_ptr, value, value->vtable);
if (use_cardtable) {
*(void**)field_ptr = value;
if (ptr_in_nursery (value))
return;
}
DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr));
+ if (value)
+ binary_protocol_wbarrier (slot_ptr, value, value->vtable);
if (use_cardtable) {
*(void**)slot_ptr = value;
if (ptr_in_nursery (value))
return;
}
+#ifdef SGEN_BINARY_PROTOCOL
+ {
+ int i;
+ for (i = 0; i < count; ++i) {
+ gpointer dest = (gpointer*)dest_ptr + i;
+ gpointer obj = *((gpointer*)src_ptr + i);
+ if (obj)
+ binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
+ }
+ }
+#endif
+
if (use_cardtable) {
gpointer *dest = dest_ptr;
gpointer *src = src_ptr;
}
}
+#ifdef SGEN_BINARY_PROTOCOL
+#undef HANDLE_PTR
+#define HANDLE_PTR(ptr,obj) do { \
+ gpointer o = *(gpointer*)(ptr); \
+ if ((o)) { \
+ gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
+ binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
+ } \
+ } while (0)
+
+static void
+scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
+{
+#define SCAN_OBJECT_NOVTABLE
+#include "sgen-scan-object.h"
+}
+#endif
void
mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
{
RememberedSet *rs;
- size_t size = count * mono_class_value_size (klass, NULL);
+ size_t element_size = mono_class_value_size (klass, NULL);
+ size_t size = count * element_size;
TLAB_ACCESS_INIT;
HEAVY_STAT (++stat_wbarrier_value_copy);
g_assert (klass->valuetype);
+#ifdef SGEN_BINARY_PROTOCOL
+ {
+ int i;
+ for (i = 0; i < count; ++i) {
+ scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
+ (char*)src + i * element_size - sizeof (MonoObject),
+ (mword) klass->gc_descr);
+ }
+ }
+#endif
if (use_cardtable) {
#ifdef DISABLE_CRITICAL_REGION
LOCK_GC;
*(rs->store_next++) = (mword)dest | REMSET_VTYPE;
*(rs->store_next++) = (mword)klass->gc_descr;
*(rs->store_next++) = (mword)count;
- *(rs->store_next++) = (mword)size;
+ *(rs->store_next++) = (mword)element_size;
UNLOCK_GC;
return;
}
*(rs->store_next++) = (mword)dest | REMSET_VTYPE;
*(rs->store_next++) = (mword)klass->gc_descr;
*(rs->store_next++) = (mword)count;
- *(rs->store_next++) = (mword)size;
+ *(rs->store_next++) = (mword)element_size;
UNLOCK_GC;
}
}
DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
size = mono_object_class (obj)->instance_size;
LOCK_GC;
+#ifdef SGEN_BINARY_PROTOCOL
+ scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
+#endif
/* do not copy the sync state */
mono_gc_memmove ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
size - sizeof (MonoObject));
char *env;
char **opts, **ptr;
char *major_collector_opt = NULL;
- struct sigaction sinfo;
glong max_heap = 0;
+ glong soft_limit = 0;
int num_workers;
int result;
+ int dummy;
do {
result = InterlockedCompareExchange (&gc_initialized, -1, 0);
return;
case -1:
/* being inited by another thread */
- usleep (1000);
+ g_usleep (1000);
break;
case 0:
/* we will init it */
LOCK_INIT (gc_mutex);
pagesize = mono_pagesize ();
- gc_debug_file = stdout;
+ gc_debug_file = stderr;
cb.thread_register = sgen_thread_register;
cb.thread_unregister = sgen_thread_unregister;
cb.thread_attach = sgen_thread_attach;
- cb.mono_method_is_critical = is_critical_method;
+ cb.mono_method_is_critical = (gpointer)is_critical_method;
+ cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
mono_threads_init (&cb, sizeof (SgenThreadInfo));
* it inits the small id which is required for hazard pointer
* operations.
*/
- suspend_ack_semaphore_ptr = &suspend_ack_semaphore;
- MONO_SEM_INIT (&suspend_ack_semaphore, 0);
-
- sigfillset (&sinfo.sa_mask);
- sinfo.sa_flags = SA_RESTART | SA_SIGINFO;
- sinfo.sa_sigaction = suspend_handler;
- if (sigaction (suspend_signal_num, &sinfo, NULL) != 0) {
- g_error ("failed sigaction");
- }
-
- sinfo.sa_handler = restart_handler;
- if (sigaction (restart_signal_num, &sinfo, NULL) != 0) {
- g_error ("failed sigaction");
- }
-
- sigfillset (&suspend_signal_mask);
- sigdelset (&suspend_signal_mask, restart_signal_num);
+ mono_sgen_os_init ();
- mono_thread_info_attach (&sinfo);
+ mono_thread_info_attach (&dummy);
if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
mono_sgen_marksweep_init (&major_collector);
}
continue;
}
+ if (g_str_has_prefix (opt, "soft-heap-limit=")) {
+ opt = strchr (opt, '=') + 1;
+ if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
+ if (soft_limit <= 0) {
+ fprintf (stderr, "soft-heap-limit must be positive.\n");
+ exit (1);
+ }
+ } else {
+ fprintf (stderr, "soft-heap-limit must be an integer.\n");
+ exit (1);
+ }
+ continue;
+ }
if (g_str_has_prefix (opt, "workers=")) {
long val;
char *endptr;
if (!(major_collector.handle_gc_param && major_collector.handle_gc_param (opt))) {
fprintf (stderr, "MONO_GC_PARAMS must be a comma-delimited list of one or more of the following:\n");
fprintf (stderr, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
+ fprintf (stderr, " soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par' or `copying')\n");
fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
nursery_size = DEFAULT_NURSERY_SIZE;
minor_collection_allowance = MIN_MINOR_COLLECTION_ALLOWANCE;
- init_heap_size_limits (max_heap);
+ init_heap_size_limits (max_heap, soft_limit);
alloc_nursery ();
gc_debug_file = stderr;
g_free (rf);
}
+ } else if (!strcmp (opt, "print-allowance")) {
+ debug_print_allowance = TRUE;
} else if (!strcmp (opt, "collect-before-allocs")) {
collect_before_allocs = 1;
} else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
fprintf (stderr, " disable-major\n");
fprintf (stderr, " xdomain-checks\n");
fprintf (stderr, " clear-at-gc\n");
+ fprintf (stderr, " print-allowance\n");
exit (1);
}
}
gc_initialized = 1;
}
-int
-mono_gc_get_suspend_signal (void)
-{
- return suspend_signal_num;
-}
-
enum {
ATYPE_NORMAL,
ATYPE_VECTOR,
mono_mb_emit_byte (mb, CEE_ADD);
mono_mb_emit_stloc (mb, new_next_var);
- /* tlab_next = new_next */
- mono_mb_emit_ldloc (mb, tlab_next_addr_var);
- mono_mb_emit_ldloc (mb, new_next_var);
- mono_mb_emit_byte (mb, CEE_STIND_I);
-
/* if (G_LIKELY (new_next < tlab_temp_end)) */
mono_mb_emit_ldloc (mb, new_next_var);
EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset);
/* FIXME: Memory barrier */
+ /* tlab_next = new_next */
+ mono_mb_emit_ldloc (mb, tlab_next_addr_var);
+ mono_mb_emit_ldloc (mb, new_next_var);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ /*The tlab store must be visible before the the vtable store. This could be replaced with a DDS but doing it with IL would be tricky. */
+ mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
+ mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
+
/* *p = vtable; */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_byte (mb, CEE_STIND_I);
}
+ /*
+ We must make sure both vtable and max_length are globaly visible before returning to managed land.
+ */
+ mono_mb_emit_byte ((mb), MONO_CUSTOM_PREFIX);
+ mono_mb_emit_op (mb, CEE_MONO_MEMORY_BARRIER, StoreStoreBarrier);
+
/* return p */
mono_mb_emit_ldloc (mb, p_var);
mono_mb_emit_byte (mb, CEE_RET);
return ATYPE_NUM;
}
+static void
+emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
+{
+ memset (nursery_check_return_labels, 0, sizeof (int) * 3);
+#ifdef SGEN_ALIGN_NURSERY
+ // if (ptr_in_nursery (ptr)) return;
+ /*
+ * Masking out the bits might be faster, but we would have to use 64 bit
+ * immediates, which might be slower.
+ */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_byte (mb, CEE_SHR_UN);
+ mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
+ nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
+
+ // if (!ptr_in_nursery (*ptr)) return;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_byte (mb, CEE_SHR_UN);
+ mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
+ nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
+#else
+ int label_continue1, label_continue2;
+ int dereferenced_var;
+
+ // if (ptr < (nursery_start)) goto continue;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_start);
+ label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
+
+ // if (ptr >= nursery_end)) goto continue;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_end);
+ label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
+
+ // Otherwise return
+ nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
+
+ // continue:
+ mono_mb_patch_branch (mb, label_continue_1);
+ mono_mb_patch_branch (mb, label_continue_2);
+
+ // Dereference and store in local var
+ dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_stloc (mb, dereferenced_var);
+
+ // if (*ptr < nursery_start) return;
+ mono_mb_emit_ldloc (mb, dereferenced_var);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_start);
+ nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
+
+ // if (*ptr >= nursery_end) return;
+ mono_mb_emit_ldloc (mb, dereferenced_var);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_end);
+ nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
+#endif
+}
MonoMethod*
mono_gc_get_write_barrier (void)
MonoMethodBuilder *mb;
MonoMethodSignature *sig;
#ifdef MANAGED_WBARRIER
- int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
-#ifndef SGEN_ALIGN_NURSERY
- int label_continue_1, label_continue_2, label_no_wb_5;
- int dereferenced_var;
-#endif
+ int i, nursery_check_labels [3];
+ int label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
int buffer_var, buffer_index_var, dummy_var;
#ifdef HAVE_KW_THREAD
#endif
#endif
- g_assert (!use_cardtable);
-
// FIXME: Maybe create a separate version for ctors (the branch would be
// correctly predicted more times)
if (write_barrier_method)
mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
#ifdef MANAGED_WBARRIER
- if (mono_runtime_has_tls_get ()) {
-#ifdef SGEN_ALIGN_NURSERY
- // if (ptr_in_nursery (ptr)) return;
+ if (use_cardtable) {
+ emit_nursery_check (mb, nursery_check_labels);
/*
- * Masking out the bits might be faster, but we would have to use 64 bit
- * immediates, which might be slower.
- */
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
- mono_mb_emit_byte (mb, CEE_SHR_UN);
- mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
- label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BEQ);
-
- // if (!ptr_in_nursery (*ptr)) return;
+ addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
+ *addr = 1;
+
+ sgen_cardtable:
+ LDC_PTR sgen_cardtable
+
+ address >> CARD_BITS
+ LDARG_0
+ LDC_I4 CARD_BITS
+ SHR_UN
+ if (SGEN_HAVE_OVERLAPPING_CARDS) {
+ LDC_PTR card_table_mask
+ AND
+ }
+ AND
+ ldc_i4_1
+ stind_i1
+ */
+ mono_mb_emit_ptr (mb, sgen_cardtable);
mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_byte (mb, CEE_LDIND_I);
- mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_icon (mb, CARD_BITS);
mono_mb_emit_byte (mb, CEE_SHR_UN);
- mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
- label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
-#else
-
- // if (ptr < (nursery_start)) goto continue;
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_ptr (mb, (gpointer) nursery_start);
- label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
-
- // if (ptr >= nursery_end)) goto continue;
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_ptr (mb, (gpointer) nursery_end);
- label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
-
- // Otherwise return
- label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
-
- // continue:
- mono_mb_patch_branch (mb, label_continue_1);
- mono_mb_patch_branch (mb, label_continue_2);
-
- // Dereference and store in local var
- dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_byte (mb, CEE_LDIND_I);
- mono_mb_emit_stloc (mb, dereferenced_var);
-
- // if (*ptr < nursery_start) return;
- mono_mb_emit_ldloc (mb, dereferenced_var);
- mono_mb_emit_ptr (mb, (gpointer) nursery_start);
- label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
+#ifdef SGEN_HAVE_OVERLAPPING_CARDS
+ mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
+ mono_mb_emit_byte (mb, CEE_AND);
+#endif
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_icon (mb, 1);
+ mono_mb_emit_byte (mb, CEE_STIND_I1);
- // if (*ptr >= nursery_end) return;
- mono_mb_emit_ldloc (mb, dereferenced_var);
- mono_mb_emit_ptr (mb, (gpointer) nursery_end);
- label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
+ // return;
+ for (i = 0; i < 3; ++i) {
+ if (nursery_check_labels [i])
+ mono_mb_patch_branch (mb, nursery_check_labels [i]);
+ }
+ mono_mb_emit_byte (mb, CEE_RET);
+ } else if (mono_runtime_has_tls_get ()) {
+ emit_nursery_check (mb, nursery_check_labels);
-#endif
// if (ptr >= stack_end) goto need_wb;
mono_mb_emit_ldarg (mb, 0);
EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
mono_mb_emit_byte (mb, CEE_STIND_I);
// return;
- mono_mb_patch_branch (mb, label_no_wb_1);
- mono_mb_patch_branch (mb, label_no_wb_2);
+ for (i = 0; i < 3; ++i) {
+ if (nursery_check_labels [i])
+ mono_mb_patch_branch (mb, nursery_check_labels [i]);
+ }
mono_mb_patch_branch (mb, label_no_wb_3);
mono_mb_patch_branch (mb, label_no_wb_4);
-#ifndef SGEN_ALIGN_NURSERY
- mono_mb_patch_branch (mb, label_no_wb_5);
-#endif
mono_mb_emit_byte (mb, CEE_RET);
// slow path
mono_mb_patch_branch (mb, label_slow_path);
- }
-#endif
- mono_mb_emit_ldarg (mb, 0);
- mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
- mono_mb_emit_byte (mb, CEE_RET);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
+ mono_mb_emit_byte (mb, CEE_RET);
+ } else
+#endif
+ {
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
+ mono_mb_emit_byte (mb, CEE_RET);
+ }
res = mono_mb_create_method (mb, sig, 16);
mono_mb_free (mb);
projects / mono.git / blobdiff