static void *empty_blocks = NULL;
static size_t num_empty_blocks = 0;
+/*
+ * We can iterate the block list also while sweep is in progress but we
+ * need to account for blocks that will be checked for sweeping and even
+ * freed in the process.
+ */
#define FOREACH_BLOCK_NO_LOCK(bl) { \
volatile gpointer *slot; \
- SGEN_ASSERT (0, !sweep_in_progress (), "Can't iterate blocks while sweep is in progress."); \
SGEN_ARRAY_LIST_FOREACH_SLOT (&allocated_blocks, slot) { \
- (bl) = BLOCK_UNTAG (*slot);
+ (bl) = BLOCK_UNTAG (*slot); \
+ if (!(bl)) \
+ continue;
#define FOREACH_BLOCK_HAS_REFERENCES_NO_LOCK(bl,hr) { \
volatile gpointer *slot; \
- SGEN_ASSERT (0, !sweep_in_progress (), "Can't iterate blocks while sweep is in progress."); \
SGEN_ARRAY_LIST_FOREACH_SLOT (&allocated_blocks, slot) { \
(bl) = (MSBlockInfo *) (*slot); \
+ if (!(bl)) \
+ continue; \
(hr) = BLOCK_IS_TAGGED_HAS_REFERENCES ((bl)); \
(bl) = BLOCK_UNTAG ((bl));
#define END_FOREACH_BLOCK_NO_LOCK } SGEN_ARRAY_LIST_END_FOREACH_SLOT; }
static guint64 stat_major_blocks_freed = 0;
static guint64 stat_major_blocks_lazy_swept = 0;
-#if SIZEOF_VOID_P != 8
static guint64 stat_major_blocks_freed_ideal = 0;
static guint64 stat_major_blocks_freed_less_ideal = 0;
static guint64 stat_major_blocks_freed_individual = 0;
static guint64 stat_major_blocks_alloced_less_ideal = 0;
-#endif
#ifdef SGEN_COUNT_NUMBER_OF_MAJOR_OBJECTS_MARKED
static guint64 num_major_objects_marked = 0;
{
char *start;
if (nursery_align)
- start = (char *)sgen_alloc_os_memory_aligned (nursery_size, nursery_align, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), "nursery");
+ start = (char *)sgen_alloc_os_memory_aligned (nursery_size, nursery_align, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), "nursery", MONO_MEM_ACCOUNT_SGEN_NURSERY);
else
- start = (char *)sgen_alloc_os_memory (nursery_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), "nursery");
+ start = (char *)sgen_alloc_os_memory (nursery_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), "nursery", MONO_MEM_ACCOUNT_SGEN_NURSERY);
return start;
}
for (;;) {
p = (char *)sgen_alloc_os_memory_aligned (MS_BLOCK_SIZE * alloc_num, MS_BLOCK_SIZE,
(SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE),
- alloc_num == 1 ? "major heap section" : NULL);
+ alloc_num == 1 ? "major heap section" : NULL, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
if (p)
break;
alloc_num >>= 1;
add_free_block (free_blocks, size_index, info);
- /*
- * Adding to the allocated_blocks array is racy with the removal of nulls when
- * sweeping. We wait for sweep to finish to avoid that.
- *
- * The memory barrier here and in `sweep_job_func()` are required because we need
- * `allocated_blocks` synchronized between this and the sweep thread.
- */
- major_finish_sweep_checking ();
- mono_memory_barrier ();
-
sgen_array_list_add (&allocated_blocks, BLOCK_TAG (info), 0, FALSE);
SGEN_ATOMIC_ADD_P (num_major_sections, 1);
}
static gboolean
-ptr_is_from_pinned_alloc (char *ptr)
+ptr_is_in_major_block (char *ptr, char **start, gboolean *pinned)
{
MSBlockInfo *block;
FOREACH_BLOCK_NO_LOCK (block) {
- if (ptr >= MS_BLOCK_FOR_BLOCK_INFO (block) && ptr <= MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE)
- return block->pinned;
+ if (ptr >= MS_BLOCK_FOR_BLOCK_INFO (block) && ptr <= MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE) {
+ int count = MS_BLOCK_FREE / block->obj_size;
+ int i;
+
+ if (start)
+ *start = NULL;
+ for (i = 0; i <= count; ++i) {
+ if (ptr >= (char*)MS_BLOCK_OBJ (block, i) && ptr < (char*)MS_BLOCK_OBJ (block, i + 1)) {
+ if (start)
+ *start = (char *)MS_BLOCK_OBJ (block, i);
+ break;
+ }
+ }
+ if (pinned)
+ *pinned = block->pinned;
+ return TRUE;
+ }
} END_FOREACH_BLOCK_NO_LOCK;
return FALSE;
}
+static gboolean
+ptr_is_from_pinned_alloc (char *ptr)
+{
+ gboolean pinned;
+ if (ptr_is_in_major_block (ptr, NULL, &pinned))
+ return pinned;
+ return FALSE;
+}
+
static void
ensure_can_access_block_free_list (MSBlockInfo *block)
{
*as pinned alloc is requested by the runtime.
*/
if (!res) {
- sgen_perform_collection (0, GENERATION_OLD, "pinned alloc failure", TRUE);
+ sgen_perform_collection (0, GENERATION_OLD, "pinned alloc failure", TRUE, TRUE);
res = alloc_obj (vtable, size, TRUE, has_references);
}
return (GCObject *)res;
static gboolean
major_ptr_is_in_non_pinned_space (char *ptr, char **start)
{
- MSBlockInfo *block;
-
- FOREACH_BLOCK_NO_LOCK (block) {
- if (ptr >= MS_BLOCK_FOR_BLOCK_INFO (block) && ptr <= MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE) {
- int count = MS_BLOCK_FREE / block->obj_size;
- int i;
-
- *start = NULL;
- for (i = 0; i <= count; ++i) {
- if (ptr >= (char*)MS_BLOCK_OBJ (block, i) && ptr < (char*)MS_BLOCK_OBJ (block, i + 1)) {
- *start = (char *)MS_BLOCK_OBJ (block, i);
- break;
- }
- }
- return !block->pinned;
- }
- } END_FOREACH_BLOCK_NO_LOCK;
+ gboolean pinned;
+ if (ptr_is_in_major_block (ptr, start, &pinned))
+ return !pinned;
return FALSE;
}
gboolean pinned = flags & ITERATE_OBJECTS_PINNED;
MSBlockInfo *block;
+ /* No actual sweeping will take place if we are in the middle of a major collection. */
major_finish_sweep_checking ();
FOREACH_BLOCK_NO_LOCK (block) {
int count = MS_BLOCK_FREE / block->obj_size;
continue;
if (!block->pinned && !non_pinned)
continue;
- if (sweep && lazy_sweep) {
+ if (sweep && lazy_sweep && !block_is_swept_or_marking (block)) {
sweep_block (block);
SGEN_ASSERT (6, block->state == BLOCK_STATE_SWEPT, "Block must be swept after sweeping");
}
for (i = 0; i < count; ++i) {
void **obj = (void**) MS_BLOCK_OBJ (block, i);
- /*
- * We've finished sweep checking, but if we're sweeping lazily and
- * the flags don't require us to sweep, the block might still need
- * sweeping. In that case, we need to consult the mark bits to tell
- * us whether an object slot is live.
- */
- if (!block_is_swept_or_marking (block)) {
- int word, bit;
- SGEN_ASSERT (6, !sweep && block->state == BLOCK_STATE_NEED_SWEEPING, "Has sweeping not finished?");
- MS_CALC_MARK_BIT (word, bit, obj);
- if (!MS_MARK_BIT (block, word, bit))
- continue;
- }
if (MS_OBJ_ALLOCED (obj, block))
callback ((GCObject*)obj, block->obj_size, data);
}
return FALSE;
}
-#define LOAD_VTABLE SGEN_LOAD_VTABLE
-
-#define MS_MARK_OBJECT_AND_ENQUEUE_CHECKED(obj,desc,block,queue) do { \
- int __word, __bit; \
- MS_CALC_MARK_BIT (__word, __bit, (obj)); \
- if (!MS_MARK_BIT ((block), __word, __bit) && MS_OBJ_ALLOCED ((obj), (block))) { \
- MS_SET_MARK_BIT ((block), __word, __bit); \
- if (sgen_gc_descr_has_references (desc)) \
- GRAY_OBJECT_ENQUEUE ((queue), (obj), (desc)); \
- binary_protocol_mark ((obj), (gpointer)LOAD_VTABLE ((obj)), sgen_safe_object_get_size ((obj))); \
- INC_NUM_MAJOR_OBJECTS_MARKED (); \
- } \
- } while (0)
#define MS_MARK_OBJECT_AND_ENQUEUE(obj,desc,block,queue) do { \
int __word, __bit; \
MS_CALC_MARK_BIT (__word, __bit, (obj)); \
MS_SET_MARK_BIT ((block), __word, __bit); \
if (sgen_gc_descr_has_references (desc)) \
GRAY_OBJECT_ENQUEUE ((queue), (obj), (desc)); \
- binary_protocol_mark ((obj), (gpointer)LOAD_VTABLE ((obj)), sgen_safe_object_get_size ((obj))); \
+ binary_protocol_mark ((obj), (gpointer)SGEN_LOAD_VTABLE ((obj)), sgen_safe_object_get_size ((obj))); \
INC_NUM_MAJOR_OBJECTS_MARKED (); \
} \
} while (0)
/* gcc 4.2.1 from xcode4 crashes on sgen_card_table_get_card_address () when this is enabled */
#if defined(PLATFORM_MACOSX)
-#define GCC_VERSION (__GNUC__ * 10000 \
- + __GNUC_MINOR__ * 100 \
- + __GNUC_PATCHLEVEL__)
-#if GCC_VERSION <= 40300
+#if MONO_GNUC_VERSION <= 40300
#undef PREFETCH_CARDS
#endif
#endif
if (first_entry == last_entry)
return;
- block->has_pinned = TRUE;
-
entry = sgen_pinning_get_entry (first_entry);
end = sgen_pinning_get_entry (last_entry);
if (index == last_index)
continue;
obj = MS_BLOCK_OBJ (block, index);
- MS_MARK_OBJECT_AND_ENQUEUE_CHECKED (obj, sgen_obj_get_descriptor (obj), block, queue);
+ if (!MS_OBJ_ALLOCED (obj, block))
+ continue;
+ MS_MARK_OBJECT_AND_ENQUEUE (obj, sgen_obj_get_descriptor (obj), block, queue);
+ sgen_pin_stats_register_object (obj, GENERATION_OLD);
last_index = index;
}
+
+ /*
+ * There might have been potential pinning "pointers" into this block, but none of
+ * them pointed to occupied slots, in which case we don't have to pin the block.
+ */
+ if (last_index >= 0)
+ block->has_pinned = TRUE;
}
static inline void
{
SGEN_ASSERT (6, block->state == expected_state, "Block state incorrect before set");
block->state = new_state;
+ binary_protocol_block_set_state (block, MS_BLOCK_SIZE, expected_state, new_state);
}
/*
for (j = 0; j < num_block_obj_sizes; ++j)
free_blocks [j] = NULL;
}
+
+ sgen_array_list_remove_nulls (&allocated_blocks);
}
static void sweep_finish (void);
}
done:
+ /*
+ * Once the block is written back without the checking bit other threads are
+ * free to access it. Make sure the block state is visible before we write it
+ * back.
+ */
+ mono_memory_write_barrier ();
*block_slot = tagged_block;
return !!tagged_block;
}
sweep_blocks_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
{
volatile gpointer *slot;
+ MSBlockInfo *bl;
SGEN_ARRAY_LIST_FOREACH_SLOT (&allocated_blocks, slot) {
- sweep_block (BLOCK_UNTAG (*slot));
+ bl = BLOCK_UNTAG (*slot);
+ if (bl)
+ sweep_block (bl);
} SGEN_ARRAY_LIST_END_FOREACH_SLOT;
mono_memory_write_barrier ();
}
}
- sgen_array_list_remove_nulls (&allocated_blocks);
-
/*
* Concurrently sweep all the blocks to reduce workload during minor
* pauses where we need certain blocks to be swept. At the start of
static void
count_nonpinned_callback (GCObject *obj, size_t size, void *data)
{
- GCVTable vtable = LOAD_VTABLE (obj);
+ GCVTable vtable = SGEN_LOAD_VTABLE (obj);
if (SGEN_VTABLE_HAS_REFERENCES (vtable))
++count_nonpinned_ref;
static void
count_pinned_callback (GCObject *obj, size_t size, void *data)
{
- GCVTable vtable = LOAD_VTABLE (obj);
+ GCVTable vtable = SGEN_LOAD_VTABLE (obj);
if (SGEN_VTABLE_HAS_REFERENCES (vtable))
++count_pinned_ref;
SGEN_ASSERT (0, num_blocks == index, "Why did the freelist change ?");
- qsort (evacuated_blocks, num_blocks, sizeof (gpointer), block_usage_comparer);
+ sgen_qsort (evacuated_blocks, num_blocks, sizeof (gpointer), block_usage_comparer);
/*
* Form a new freelist with the fullest blocks. These blocks will also be
#endif
}
-#if SIZEOF_VOID_P != 8
static int
compare_pointers (const void *va, const void *vb) {
char *a = *(char**)va, *b = *(char**)vb;
return 1;
return 0;
}
-#endif
/*
* This is called with sweep completed and the world stopped.
*/
static void
-major_free_swept_blocks (size_t allowance)
+major_free_swept_blocks (size_t section_reserve)
{
- /* FIXME: This is probably too much. It's assuming all objects are small. */
- size_t section_reserve = allowance / MS_BLOCK_SIZE;
-
SGEN_ASSERT (0, sweep_state == SWEEP_STATE_SWEPT, "Sweeping must have finished before freeing blocks");
#ifdef TARGET_WIN32
return;
#endif
-#if SIZEOF_VOID_P != 8
{
int i, num_empty_blocks_orig, num_blocks, arr_length;
void *block;
* we're iterating.
*/
int j;
- sgen_free_os_memory (empty_block_arr [first], MS_BLOCK_SIZE * num_blocks, SGEN_ALLOC_HEAP);
+ sgen_free_os_memory (empty_block_arr [first], MS_BLOCK_SIZE * num_blocks, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
for (j = first; j <= d; ++j)
empty_block_arr [j] = NULL;
dest = first;
*/
if (num_empty_blocks <= num_major_sections)
return;
-#endif
while (num_empty_blocks > section_reserve) {
void *next = *(void**)empty_blocks;
- sgen_free_os_memory (empty_blocks, MS_BLOCK_SIZE, SGEN_ALLOC_HEAP);
+ sgen_free_os_memory (empty_blocks, MS_BLOCK_SIZE, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
empty_blocks = next;
/*
* Needs not be atomic because this is running
--num_empty_blocks;
++stat_major_blocks_freed;
-#if SIZEOF_VOID_P != 8
++stat_major_blocks_freed_individual;
-#endif
}
}
/*
* This callback is used to clear cards, move cards to the shadow table and do counting.
*/
+static void
+major_iterate_block_ranges (sgen_cardtable_block_callback callback)
+{
+ MSBlockInfo *block;
+ gboolean has_references;
+
+ FOREACH_BLOCK_HAS_REFERENCES_NO_LOCK (block, has_references) {
+ if (has_references)
+ callback ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE);
+ } END_FOREACH_BLOCK_NO_LOCK;
+}
+
static void
major_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
{
major_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx)
{
MSBlockInfo *block;
- gboolean has_references;
+ gboolean has_references, was_sweeping, skip_scan;
if (!concurrent_mark)
g_assert (scan_type == CARDTABLE_SCAN_GLOBAL);
- major_finish_sweep_checking ();
+ if (scan_type != CARDTABLE_SCAN_GLOBAL)
+ SGEN_ASSERT (0, !sweep_in_progress (), "Sweep should be finished when we scan mod union card table");
+ was_sweeping = sweep_in_progress ();
+
binary_protocol_major_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
FOREACH_BLOCK_HAS_REFERENCES_NO_LOCK (block, has_references) {
#ifdef PREFETCH_CARDS
if (!has_references)
continue;
+ skip_scan = FALSE;
- scan_card_table_for_block (block, scan_type, ctx);
+ if (scan_type == CARDTABLE_SCAN_GLOBAL) {
+ gpointer *card_start = (gpointer*) sgen_card_table_get_card_scan_address ((mword)MS_BLOCK_FOR_BLOCK_INFO (block));
+ gboolean has_dirty_cards = FALSE;
+ int i;
+ for (i = 0; i < CARDS_PER_BLOCK / sizeof(gpointer); i++) {
+ if (card_start [i]) {
+ has_dirty_cards = TRUE;
+ break;
+ }
+ }
+ if (!has_dirty_cards) {
+ skip_scan = TRUE;
+ } else {
+ /*
+ * After the start of the concurrent collections, blocks change state
+ * to marking. We should not sweep it in that case. We can't race with
+ * sweep start since we are in a nursery collection. Also avoid CAS-ing
+ */
+ if (sweep_in_progress ()) {
+ skip_scan = !ensure_block_is_checked_for_sweeping (__index, TRUE, NULL);
+ } else if (was_sweeping) {
+ /* Recheck in case sweep finished after dereferencing the slot */
+ skip_scan = *sgen_array_list_get_slot (&allocated_blocks, __index) == 0;
+ }
+ }
+ }
+ if (!skip_scan)
+ scan_card_table_for_block (block, scan_type, ctx);
} END_FOREACH_BLOCK_NO_LOCK;
binary_protocol_major_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
}
mono_counters_register ("# major blocks allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_alloced);
mono_counters_register ("# major blocks freed", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed);
mono_counters_register ("# major blocks lazy swept", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_lazy_swept);
-#if SIZEOF_VOID_P != 8
mono_counters_register ("# major blocks freed ideally", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed_ideal);
mono_counters_register ("# major blocks freed less ideally", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed_less_ideal);
mono_counters_register ("# major blocks freed individually", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed_individual);
mono_counters_register ("# major blocks allocated less ideally", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_alloced_less_ideal);
-#endif
collector->section_size = MAJOR_SECTION_SIZE;
collector->pin_major_object = pin_major_object;
collector->scan_card_table = major_scan_card_table;
collector->iterate_live_block_ranges = major_iterate_live_block_ranges;
+ collector->iterate_block_ranges = major_iterate_block_ranges;
if (is_concurrent) {
collector->update_cardtable_mod_union = update_cardtable_mod_union;
collector->get_cardtable_mod_union_for_reference = major_get_cardtable_mod_union_for_reference;