-/*
- * sgen-cardtable.c: Card table implementation for sgen
+/**
+ * \file
+ * Card table implementation for sgen
*
* Author:
* Rodrigo Kumpera (rkumpera@novell.com)
* Copyright 2011 Xamarin Inc (http://www.xamarin.com)
* Copyright (C) 2012 Xamarin Inc
*
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License 2.0 as published by the Free Software Foundation;
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License 2.0 along with this library; if not, write to the Free
- * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ * Licensed under the MIT license. See LICENSE file in the project root for full license information.
*/
#include "config.h"
static guint64 large_objects;
static guint64 bloby_objects;
#endif
-static guint64 major_card_scan_time;
-static guint64 los_card_scan_time;
-
-static guint64 last_major_scan_time;
-static guint64 last_los_scan_time;
-
-static void sgen_card_tables_collect_stats (gboolean begin);
mword
sgen_card_table_number_of_cards_in_range (mword address, mword size)
static void
sgen_card_table_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
{
- gpointer *dest = dest_ptr;
- gpointer *src = src_ptr;
+ gpointer *dest = (gpointer *)dest_ptr;
+ gpointer *src = (gpointer *)src_ptr;
/*overlapping that required backward copying*/
if (src < dest && (src + count) > dest) {
{
size_t size = count * element_size;
-#ifdef DISABLE_CRITICAL_REGION
- LOCK_GC;
-#else
TLAB_ACCESS_INIT;
ENTER_CRITICAL_REGION;
-#endif
+
mono_gc_memmove_atomic (dest, src, size);
sgen_card_table_mark_range ((mword)dest, size);
-#ifdef DISABLE_CRITICAL_REGION
- UNLOCK_GC;
-#else
+
EXIT_CRITICAL_REGION;
-#endif
}
static void
{
size_t size = sgen_client_par_object_get_size (SGEN_LOAD_VTABLE_UNCHECKED (obj), obj);
-#ifdef DISABLE_CRITICAL_REGION
- LOCK_GC;
-#else
TLAB_ACCESS_INIT;
ENTER_CRITICAL_REGION;
-#endif
+
mono_gc_memmove_aligned ((char*)obj + SGEN_CLIENT_OBJECT_HEADER_SIZE, (char*)src + SGEN_CLIENT_OBJECT_HEADER_SIZE,
size - SGEN_CLIENT_OBJECT_HEADER_SIZE);
sgen_card_table_mark_range ((mword)obj, size);
-#ifdef DISABLE_CRITICAL_REGION
- UNLOCK_GC;
-#else
+
EXIT_CRITICAL_REGION;
-#endif
}
static void
sgen_card_table_mark_address ((mword)ptr);
}
+static void
+sgen_card_table_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
+{
+ GCObject **dest = (GCObject **)_dest;
+ GCObject **src = (GCObject **)_src;
+
+ size_t nursery_bits = DEFAULT_NURSERY_BITS;
+ char *start = sgen_nursery_start;
+ G_GNUC_UNUSED char *end = sgen_nursery_end;
+
+ /*
+ * It's cardtable theory time!
+ * Our cardtable scanning code supports marking any card that an object/valuetype belongs to.
+ * This function is supposed to be used to copy a range that fully belongs to a single type.
+ * It must not be used, for example, to copy 2 adjacent VTs in an array.
+ */
+ volatile guint8 *card_address = (volatile guint8 *)sgen_card_table_get_card_address ((mword)dest);
+ while (size) {
+ GCObject *value = *src;
+ *dest = value;
+ if (SGEN_PTR_IN_NURSERY (value, nursery_bits, start, end) || concurrent_collection_in_progress) {
+ *card_address = 1;
+ sgen_dummy_use (value);
+ }
+ ++src;
+ ++dest;
+ size -= SIZEOF_VOID_P;
+ }
+}
+
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
guint8 *sgen_shadow_cardtable;
static gboolean
sgen_card_table_find_address_with_cards (char *cards_start, guint8 *cards, char *addr)
{
- cards_start = sgen_card_table_align_pointer (cards_start);
+ cards_start = (char *)sgen_card_table_align_pointer (cards_start);
return cards [(addr - cards_start) >> CARD_BITS];
}
update_mod_union (guint8 *dest, guint8 *start_card, size_t num_cards)
{
int i;
- for (i = 0; i < num_cards; ++i)
- dest [i] |= start_card [i];
+ /* Marking from another thread can happen while we mark here */
+ for (i = 0; i < num_cards; ++i) {
+ if (start_card [i])
+ dest [i] = 1;
+ }
}
guint8*
sgen_card_table_alloc_mod_union (char *obj, mword obj_size)
{
size_t num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj, obj_size);
- guint8 *mod_union = sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
+ guint8 *mod_union = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
memset (mod_union, 0, num_cards);
return mod_union;
}
*out_num_cards = num_cards;
}
+/* Preclean cards and saves the cards that need to be scanned afterwards in cards_preclean */
+void
+sgen_card_table_preclean_mod_union (guint8 *cards, guint8 *cards_preclean, size_t num_cards)
+{
+ size_t i;
+
+ memcpy (cards_preclean, cards, num_cards);
+ for (i = 0; i < num_cards; i++) {
+ if (cards_preclean [i]) {
+ cards [i] = 0;
+ }
+ }
+ /*
+ * When precleaning we need to make sure the card cleaning
+ * takes place before the object is scanned. If we don't
+ * do this we could finish scanning the object and, before
+ * the cleaning of the card takes place, another thread
+ * could dirty the object, mark the mod_union card only for
+ * us to clean it back, without scanning the object again.
+ */
+ mono_memory_barrier ();
+}
+
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
static void
sgen_card_table_clear_cards (void)
{
/*XXX we could do this in 2 ways. using mincore or iterating over all sections/los objects */
- sgen_major_collector_iterate_live_block_ranges (clear_cards);
+ sgen_major_collector_iterate_block_ranges (clear_cards);
sgen_los_iterate_live_block_ranges (clear_cards);
+ sgen_wbroots_iterate_live_block_ranges (clear_cards);
}
static void
-sgen_card_table_finish_minor_collection (void)
-{
- sgen_card_tables_collect_stats (FALSE);
-}
-
-static void
-sgen_card_table_scan_remsets (ScanCopyContext ctx)
+sgen_card_table_start_scan_remsets (void)
{
- SGEN_TV_DECLARE (atv);
- SGEN_TV_DECLARE (btv);
-
- sgen_card_tables_collect_stats (TRUE);
-
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
/*FIXME we should have a bit on each block/los object telling if the object have marked cards.*/
/*First we copy*/
- sgen_major_collector_iterate_live_block_ranges (move_cards_to_shadow_table);
+ sgen_major_collector_iterate_block_ranges (move_cards_to_shadow_table);
sgen_los_iterate_live_block_ranges (move_cards_to_shadow_table);
+ sgen_wbroots_iterate_live_block_ranges (move_cards_to_shadow_table);
/*Then we clear*/
sgen_card_table_clear_cards ();
#endif
- SGEN_TV_GETTIME (atv);
- sgen_get_major_collector ()->scan_card_table (FALSE, ctx);
- SGEN_TV_GETTIME (btv);
- last_major_scan_time = SGEN_TV_ELAPSED (atv, btv);
- major_card_scan_time += last_major_scan_time;
- sgen_los_scan_card_table (FALSE, ctx);
- SGEN_TV_GETTIME (atv);
- last_los_scan_time = SGEN_TV_ELAPSED (btv, atv);
- los_card_scan_time += last_los_scan_time;
}
guint8*
}
#endif
-void
-sgen_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, gboolean mod_union, ScanCopyContext ctx)
-{
- HEAVY_STAT (++large_objects);
+/*
+ * Cardtable scanning
+ */
- if (sgen_client_cardtable_scan_object (obj, block_obj_size, cards, mod_union, ctx))
- return;
+#define MWORD_MASK (sizeof (mword) - 1)
- HEAVY_STAT (++bloby_objects);
- if (cards) {
- if (sgen_card_table_is_range_marked (cards, (mword)obj, block_obj_size))
- ctx.ops->scan_object (obj, sgen_obj_get_descriptor (obj), ctx.queue);
- } else if (sgen_card_table_region_begin_scanning ((mword)obj, block_obj_size)) {
- ctx.ops->scan_object (obj, sgen_obj_get_descriptor (obj), ctx.queue);
+static inline int
+find_card_offset (mword card)
+{
+/*XXX Use assembly as this generates some pretty bad code */
+#if (defined(__i386__) || defined(__arm__)) && defined(__GNUC__)
+ return (__builtin_ffs (card) - 1) / 8;
+#elif (defined(__x86_64__) || defined(__aarch64__)) && defined(__GNUC__)
+ return (__builtin_ffsll (card) - 1) / 8;
+#elif defined(__s390x__)
+ return (__builtin_ffsll (GUINT64_TO_LE(card)) - 1) / 8;
+#else
+ int i;
+ guint8 *ptr = (guint8 *) &card;
+ for (i = 0; i < sizeof (mword); ++i) {
+ if (ptr[i])
+ return i;
}
-
- binary_protocol_card_scan (obj, sgen_safe_object_get_size (obj));
+ return 0;
+#endif
}
-#ifdef CARDTABLE_STATS
+guint8*
+sgen_find_next_card (guint8 *card_data, guint8 *end)
+{
+ mword *cards, *cards_end;
+ mword card;
-typedef struct {
- int total, marked, remarked, gc_marked;
-} card_stats;
+ while ((((mword)card_data) & MWORD_MASK) && card_data < end) {
+ if (*card_data)
+ return card_data;
+ ++card_data;
+ }
-static card_stats major_stats, los_stats;
-static card_stats *cur_stats;
+ if (card_data == end)
+ return end;
-static void
-count_marked_cards (mword start, mword size)
-{
- mword end = start + size;
- while (start <= end) {
- guint8 card = *sgen_card_table_get_card_address (start);
- ++cur_stats->total;
+ cards = (mword*)card_data;
+ cards_end = (mword*)((mword)end & ~MWORD_MASK);
+ while (cards < cards_end) {
+ card = *cards;
if (card)
- ++cur_stats->marked;
- if (card == 2)
- ++cur_stats->gc_marked;
- start += CARD_SIZE_IN_BYTES;
+ return (guint8*)cards + find_card_offset (card);
+ ++cards;
}
-}
-static void
-count_remarked_cards (mword start, mword size)
-{
- mword end = start + size;
- while (start <= end) {
- if (sgen_card_table_address_is_marked (start)) {
- ++cur_stats->remarked;
- *sgen_card_table_get_card_address (start) = 2;
- }
- start += CARD_SIZE_IN_BYTES;
+ card_data = (guint8*)cards_end;
+ while (card_data < end) {
+ if (*card_data)
+ return card_data;
+ ++card_data;
}
+
+ return end;
}
-#endif
+void
+sgen_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards, ScanCopyContext ctx)
+{
+ HEAVY_STAT (++large_objects);
-static void
-sgen_card_tables_collect_stats (gboolean begin)
-{
-#ifdef CARDTABLE_STATS
- if (begin) {
- memset (&major_stats, 0, sizeof (card_stats));
- memset (&los_stats, 0, sizeof (card_stats));
- cur_stats = &major_stats;
- sgen_major_collector_iterate_live_block_ranges (count_marked_cards);
- cur_stats = &los_stats;
- sgen_los_iterate_live_block_ranges (count_marked_cards);
- } else {
- cur_stats = &major_stats;
- sgen_major_collector_iterate_live_block_ranges (count_remarked_cards);
- cur_stats = &los_stats;
- sgen_los_iterate_live_block_ranges (count_remarked_cards);
- printf ("cards major (t %d m %d g %d r %d) los (t %d m %d g %d r %d) major_scan %.2fms los_scan %.2fms\n",
- major_stats.total, major_stats.marked, major_stats.gc_marked, major_stats.remarked,
- los_stats.total, los_stats.marked, los_stats.gc_marked, los_stats.remarked,
- last_major_scan_time / 10000.0f, last_los_scan_time / 10000.0f);
+ if (sgen_client_cardtable_scan_object (obj, cards, ctx))
+ return;
+
+ HEAVY_STAT (++bloby_objects);
+ if (cards) {
+ if (sgen_card_table_is_range_marked (cards, (mword)obj, block_obj_size))
+ ctx.ops->scan_object (obj, sgen_obj_get_descriptor (obj), ctx.queue);
+ } else if (sgen_card_table_region_begin_scanning ((mword)obj, block_obj_size)) {
+ ctx.ops->scan_object (obj, sgen_obj_get_descriptor (obj), ctx.queue);
}
-#endif
+
+ binary_protocol_card_scan (obj, sgen_safe_object_get_size (obj));
}
void
sgen_card_table_init (SgenRememberedSet *remset)
{
- sgen_cardtable = sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, SGEN_ALLOC_INTERNAL | SGEN_ALLOC_ACTIVATE, "card table");
+ sgen_cardtable = (guint8 *)sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, (SgenAllocFlags)(SGEN_ALLOC_INTERNAL | SGEN_ALLOC_ACTIVATE), "card table", MONO_MEM_ACCOUNT_SGEN_CARD_TABLE);
#ifdef SGEN_HAVE_OVERLAPPING_CARDS
- sgen_shadow_cardtable = sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, SGEN_ALLOC_INTERNAL | SGEN_ALLOC_ACTIVATE, "shadow card table");
+ sgen_shadow_cardtable = (guint8 *)sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, (SgenAllocFlags)(SGEN_ALLOC_INTERNAL | SGEN_ALLOC_ACTIVATE), "shadow card table", MONO_MEM_ACCOUNT_SGEN_SHADOW_CARD_TABLE);
#endif
#ifdef HEAVY_STATISTICS
mono_counters_register ("cardtable large objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &large_objects);
mono_counters_register ("cardtable bloby objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &bloby_objects);
#endif
- mono_counters_register ("cardtable major scan time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &major_card_scan_time);
- mono_counters_register ("cardtable los scan time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &los_card_scan_time);
-
remset->wbarrier_set_field = sgen_card_table_wbarrier_set_field;
remset->wbarrier_arrayref_copy = sgen_card_table_wbarrier_arrayref_copy;
remset->wbarrier_generic_nostore = sgen_card_table_wbarrier_generic_nostore;
remset->record_pointer = sgen_card_table_record_pointer;
- remset->scan_remsets = sgen_card_table_scan_remsets;
+ remset->start_scan_remsets = sgen_card_table_start_scan_remsets;
- remset->finish_minor_collection = sgen_card_table_finish_minor_collection;
remset->clear_cards = sgen_card_table_clear_cards;
remset->find_address = sgen_card_table_find_address;
remset->find_address_with_cards = sgen_card_table_find_address_with_cards;
+ remset->wbarrier_range_copy = sgen_card_table_wbarrier_range_copy;
need_mod_union = sgen_get_major_collector ()->is_concurrent;
}