Merge pull request #1624 from esdrubal/getprocesstimes
[mono.git] / mono / metadata / sgen-marksweep.c
index d60f8d045672488a00a5ac8ee66fd19583c3995d..db91b6773c85c5f9f898137fdb4b7d9b9b23d6cd 100644 (file)
 #include "metadata/sgen-pinning.h"
 #include "metadata/sgen-workers.h"
 
-#define SGEN_HAVE_CONCURRENT_MARK
-
-#define MS_BLOCK_SIZE_SHIFT    14      /* INT FASTENABLE */
-#define MS_BLOCK_SIZE          (1 << MS_BLOCK_SIZE_SHIFT)
+#if defined(ARCH_MIN_MS_BLOCK_SIZE) && defined(ARCH_MIN_MS_BLOCK_SIZE_SHIFT)
+#define MS_BLOCK_SIZE  ARCH_MIN_MS_BLOCK_SIZE
+#define MS_BLOCK_SIZE_SHIFT    ARCH_MIN_MS_BLOCK_SIZE_SHIFT
+#else
+#define MS_BLOCK_SIZE_SHIFT     14      /* INT FASTENABLE */
+#define MS_BLOCK_SIZE           (1 << MS_BLOCK_SIZE_SHIFT)
+#endif
 #define MAJOR_SECTION_SIZE     MS_BLOCK_SIZE
 #define CARDS_PER_BLOCK (MS_BLOCK_SIZE / CARD_SIZE_IN_BYTES)
 
 
 typedef struct _MSBlockInfo MSBlockInfo;
 struct _MSBlockInfo {
-       int obj_size;
-       int obj_size_index;
+       guint16 obj_size;
+       /*
+        * FIXME: Do we even need this? It's only used during sweep and might be worth
+        * recalculating to save the space.
+        */
+       guint16 obj_size_index;
        unsigned int pinned : 1;
        unsigned int has_references : 1;
        unsigned int has_pinned : 1;    /* means cannot evacuate */
        unsigned int is_to_space : 1;
        unsigned int swept : 1;
-       void **free_list;
-       MSBlockInfo *next_free;
-       size_t pin_queue_first_entry;
-       size_t pin_queue_last_entry;
-#ifdef SGEN_HAVE_CONCURRENT_MARK
+       void ** volatile free_list;
+       MSBlockInfo * volatile next_free;
        guint8 *cardtable_mod_union;
-#endif
        mword mark_words [MS_NUM_MARK_WORDS];
 };
 
@@ -138,17 +141,13 @@ static int fast_block_obj_size_indexes [MS_NUM_FAST_BLOCK_OBJ_SIZE_INDEXES];
 
 static gboolean *evacuate_block_obj_sizes;
 static float evacuation_threshold = 0.666f;
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 static float concurrent_evacuation_threshold = 0.666f;
 static gboolean want_evacuation = FALSE;
-#endif
 
 static gboolean lazy_sweep = TRUE;
-static gboolean have_swept;
+static gboolean have_swept = TRUE;
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 static gboolean concurrent_mark;
-#endif
 
 #define BLOCK_IS_TAGGED_HAS_REFERENCES(bl)     SGEN_POINTER_IS_TAGGED_1 ((bl))
 #define BLOCK_TAG_HAS_REFERENCES(bl)           SGEN_POINTER_TAG_1 ((bl))
@@ -310,6 +309,10 @@ ms_get_empty_block (void)
        return block;
 }
 
+/*
+ * This doesn't actually free a block immediately, but enqueues it into the `empty_blocks`
+ * list, where it will either be freed later on, or reused in nursery collections.
+ */
 static void
 ms_free_block (void *block)
 {
@@ -435,9 +438,7 @@ ms_alloc_block (int size_index, gboolean pinned, gboolean has_references)
         */
        info->is_to_space = (sgen_get_current_collection_generation () == GENERATION_OLD);
        info->swept = 1;
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        info->cardtable_mod_union = NULL;
-#endif
 
        update_heap_boundaries_for_block (info);
 
@@ -851,10 +852,8 @@ pin_major_object (char *obj, SgenGrayQueue *queue)
 {
        MSBlockInfo *block;
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        if (concurrent_mark)
                g_assert_not_reached ();
-#endif
 
        block = MS_BLOCK_FOR_OBJ (obj);
        block->has_pinned = TRUE;
@@ -863,7 +862,6 @@ pin_major_object (char *obj, SgenGrayQueue *queue)
 
 #include "sgen-major-copy-object.h"
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 static void
 major_copy_or_mark_object_with_evacuation_concurrent (void **ptr, void *obj, SgenGrayQueue *queue)
 {
@@ -898,7 +896,6 @@ major_copy_or_mark_object_with_evacuation_concurrent (void **ptr, void *obj, Sge
                }
        }
 }
-#endif
 
 static long long
 major_get_and_reset_num_major_objects_marked (void)
@@ -912,6 +909,21 @@ major_get_and_reset_num_major_objects_marked (void)
 #endif
 }
 
+#define PREFETCH_CARDS         1       /* BOOL FASTENABLE */
+#if !PREFETCH_CARDS
+#undef PREFETCH_CARDS
+#endif
+
+/* gcc 4.2.1 from xcode4 crashes on sgen_card_table_get_card_address () when this is enabled */
+#if defined(PLATFORM_MACOSX)
+#define GCC_VERSION (__GNUC__ * 10000 \
+                               + __GNUC_MINOR__ * 100 \
+                               + __GNUC_PATCHLEVEL__)
+#if GCC_VERSION <= 40300
+#undef PREFETCH_CARDS
+#endif
+#endif
+
 #ifdef HEAVY_STATISTICS
 static guint64 stat_optimized_copy;
 static guint64 stat_optimized_copy_nursery;
@@ -962,9 +974,7 @@ drain_gray_stack (ScanCopyContext ctx)
                return drain_gray_stack_no_evacuation (ctx);
 }
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 #include "sgen-marksweep-scan-object-concurrent.h"
-#endif
 
 static void
 major_copy_or_mark_object_canonical (void **ptr, SgenGrayQueue *queue)
@@ -972,27 +982,25 @@ major_copy_or_mark_object_canonical (void **ptr, SgenGrayQueue *queue)
        major_copy_or_mark_object_with_evacuation (ptr, *ptr, queue);
 }
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 static void
 major_copy_or_mark_object_concurrent_canonical (void **ptr, SgenGrayQueue *queue)
 {
        major_copy_or_mark_object_with_evacuation_concurrent (ptr, *ptr, queue);
 }
-#endif
 
 static void
-mark_pinned_objects_in_block (MSBlockInfo *block, SgenGrayQueue *queue)
+mark_pinned_objects_in_block (MSBlockInfo *block, size_t first_entry, size_t last_entry, SgenGrayQueue *queue)
 {
        void **entry, **end;
        int last_index = -1;
 
-       if (block->pin_queue_first_entry == block->pin_queue_last_entry)
+       if (first_entry == last_entry)
                return;
 
        block->has_pinned = TRUE;
 
-       entry = sgen_pinning_get_entry (block->pin_queue_first_entry);
-       end = sgen_pinning_get_entry (block->pin_queue_last_entry);
+       entry = sgen_pinning_get_entry (first_entry);
+       end = sgen_pinning_get_entry (last_entry);
 
        for (; entry < end; ++entry) {
                int index = MS_BLOCK_OBJ_INDEX (*entry, block);
@@ -1105,7 +1113,7 @@ bitcount (mword d)
 }
 
 static void
-ms_sweep (void)
+major_sweep (void)
 {
        int i;
        MSBlockInfo *block;
@@ -1115,10 +1123,8 @@ ms_sweep (void)
        int *slots_used = alloca (sizeof (int) * num_block_obj_sizes);
        int *num_blocks = alloca (sizeof (int) * num_block_obj_sizes);
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        mword total_evacuate_heap = 0;
        mword total_evacuate_saved = 0;
-#endif
 
        for (i = 0; i < num_block_obj_sizes; ++i)
                slots_available [i] = slots_used [i] = num_blocks [i] = 0;
@@ -1150,12 +1156,10 @@ ms_sweep (void)
 
                count = MS_BLOCK_FREE / block->obj_size;
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
                if (block->cardtable_mod_union) {
                        sgen_free_internal_dynamic (block->cardtable_mod_union, CARDS_PER_BLOCK, INTERNAL_MEM_CARDTABLE_MOD_UNION);
                        block->cardtable_mod_union = NULL;
                }
-#endif
 
                /* Count marked objects in the block */
                for (i = 0; i < MS_NUM_MARK_WORDS; ++i) {
@@ -1215,27 +1219,23 @@ ms_sweep (void)
                } else {
                        evacuate_block_obj_sizes [i] = FALSE;
                }
-#ifdef SGEN_HAVE_CONCURRENT_MARK
                {
                        mword total_bytes = block_obj_sizes [i] * slots_available [i];
                        total_evacuate_heap += total_bytes;
                        if (evacuate_block_obj_sizes [i])
                                total_evacuate_saved += total_bytes - block_obj_sizes [i] * slots_used [i];
                }
-#endif
        }
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        want_evacuation = (float)total_evacuate_saved / (float)total_evacuate_heap > (1 - concurrent_evacuation_threshold);
-#endif
 
        have_swept = TRUE;
 }
 
-static void
-major_sweep (void)
+static gboolean
+major_have_finished_sweeping (void)
 {
-       ms_sweep ();
+       return have_swept;
 }
 
 static int count_pinned_ref;
@@ -1289,12 +1289,26 @@ count_ref_nonref_objs (void)
 static int
 ms_calculate_block_obj_sizes (double factor, int *arr)
 {
-       double target_size = sizeof (MonoObject);
+       double target_size;
        int num_sizes = 0;
        int last_size = 0;
 
+       /*
+        * Have every possible slot size starting with the minimal
+        * object size up to and including four times that size.  Then
+        * proceed by increasing geometrically with the given factor.
+        */
+
+       for (int size = sizeof (MonoObject); size <= 4 * sizeof (MonoObject); size += SGEN_ALLOC_ALIGN) {
+               if (arr)
+                       arr [num_sizes] = size;
+               ++num_sizes;
+               last_size = size;
+       }
+       target_size = (double)last_size;
+
        do {
-               int target_count = (int)ceil (MS_BLOCK_FREE / target_size);
+               int target_count = (int)floor (MS_BLOCK_FREE / target_size);
                int size = MIN ((MS_BLOCK_FREE / target_count) & ~(SGEN_ALLOC_ALIGN - 1), SGEN_MAX_SMALL_OBJ_SIZE);
 
                if (size != last_size) {
@@ -1358,6 +1372,9 @@ major_start_major_collection (void)
 
                MONO_GC_SWEEP_END (GENERATION_OLD, TRUE);
        }
+
+       SGEN_ASSERT (0, have_swept, "Cannot start major collection without having finished sweeping");
+       have_swept = FALSE;
 }
 
 static void
@@ -1388,7 +1405,7 @@ compare_pointers (const void *va, const void *vb) {
 #endif
 
 static void
-major_have_computer_minor_collection_allowance (void)
+major_free_swept_blocks (void)
 {
        size_t section_reserve = sgen_get_minor_collection_allowance () / MS_BLOCK_SIZE;
 
@@ -1541,24 +1558,17 @@ major_have_computer_minor_collection_allowance (void)
        }
 }
 
-static void
-major_find_pin_queue_start_ends (SgenGrayQueue *queue)
-{
-       MSBlockInfo *block;
-
-       FOREACH_BLOCK (block) {
-               sgen_find_optimized_pin_queue_area (MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SKIP, MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE,
-                               &block->pin_queue_first_entry, &block->pin_queue_last_entry);
-       } END_FOREACH_BLOCK;
-}
-
 static void
 major_pin_objects (SgenGrayQueue *queue)
 {
        MSBlockInfo *block;
 
        FOREACH_BLOCK (block) {
-               mark_pinned_objects_in_block (block, queue);
+               size_t first_entry, last_entry;
+               SGEN_ASSERT (0, block->swept, "All blocks must be swept when we're pinning.");
+               sgen_find_optimized_pin_queue_area (MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SKIP, MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE,
+                               &first_entry, &last_entry);
+               mark_pinned_objects_in_block (block, first_entry, last_entry, queue);
        } END_FOREACH_BLOCK;
 }
 
@@ -1684,19 +1694,16 @@ initial_skip_card (guint8 *card_data)
 #endif
 }
 
-
-static G_GNUC_UNUSED guint8*
-skip_card (guint8 *card_data, guint8 *card_data_end)
-{
-       while (card_data < card_data_end && !*card_data)
-               ++card_data;
-       return card_data;
-}
-
 #define MS_BLOCK_OBJ_INDEX_FAST(o,b,os)        (((char*)(o) - ((b) + MS_BLOCK_SKIP)) / (os))
 #define MS_BLOCK_OBJ_FAST(b,os,i)                      ((b) + MS_BLOCK_SKIP + (os) * (i))
 #define MS_OBJ_ALLOCED_FAST(o,b)               (*(void**)(o) && (*(char**)(o) < (b) || *(char**)(o) >= (b) + MS_BLOCK_SIZE))
 
+static size_t
+card_offset (char *obj, char *base)
+{
+       return (obj - base) >> CARD_BITS;
+}
+
 static void
 major_scan_card_table (gboolean mod_union, SgenGrayQueue *queue)
 {
@@ -1704,160 +1711,138 @@ major_scan_card_table (gboolean mod_union, SgenGrayQueue *queue)
        gboolean has_references;
        ScanObjectFunc scan_func = sgen_get_current_object_ops ()->scan_object;
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        if (!concurrent_mark)
                g_assert (!mod_union);
-#else
-       g_assert (!mod_union);
-#endif
 
        FOREACH_BLOCK_HAS_REFERENCES (block, has_references) {
+#ifndef SGEN_HAVE_OVERLAPPING_CARDS
+               guint8 cards_copy [CARDS_PER_BLOCK];
+#endif
+               gboolean small_objects;
                int block_obj_size;
                char *block_start;
+               guint8 *card_data, *card_base;
+               guint8 *card_data_end;
+               char *scan_front = NULL;
+
+#ifdef PREFETCH_CARDS
+               int prefetch_index = __index + 6;
+               if (prefetch_index < allocated_blocks.next_slot) {
+                       MSBlockInfo *prefetch_block = BLOCK_UNTAG_HAS_REFERENCES (allocated_blocks.data [prefetch_index]);
+                       guint8 *prefetch_cards = sgen_card_table_get_card_scan_address ((mword)MS_BLOCK_FOR_BLOCK_INFO (prefetch_block));
+                       PREFETCH_READ (prefetch_block);
+                       PREFETCH_WRITE (prefetch_cards);
+                       PREFETCH_WRITE (prefetch_cards + 32);
+                }
+#endif
 
                if (!has_references)
                        continue;
 
                block_obj_size = block->obj_size;
-               block_start = MS_BLOCK_FOR_BLOCK_INFO (block);
+               small_objects = block_obj_size < CARD_SIZE_IN_BYTES;
 
-               if (block_obj_size >= CARD_SIZE_IN_BYTES) {
-                       guint8 *cards;
-#ifndef SGEN_HAVE_OVERLAPPING_CARDS
-                       guint8 cards_data [CARDS_PER_BLOCK];
-#endif
-                       char *obj, *end, *base;
+               block_start = MS_BLOCK_FOR_BLOCK_INFO (block);
 
-                       if (mod_union) {
-#ifdef SGEN_HAVE_CONCURRENT_MARK
-                               cards = block->cardtable_mod_union;
-                               /*
-                                * This happens when the nursery
-                                * collection that precedes finishing
-                                * the concurrent collection allocates
-                                * new major blocks.
-                                */
-                               if (!cards)
-                                       continue;
-#endif
-                       } else {
-                       /*We can avoid the extra copy since the remark cardtable was cleaned before */
+               /*
+                * This is safe in face of card aliasing for the following reason:
+                *
+                * Major blocks are 16k aligned, or 32 cards aligned.
+                * Cards aliasing happens in powers of two, so as long as major blocks are aligned to their
+                * sizes, they won't overflow the cardtable overlap modulus.
+                */
+               if (mod_union) {
+                       card_data = card_base = block->cardtable_mod_union;
+                       /*
+                        * This happens when the nursery collection that precedes finishing
+                        * the concurrent collection allocates new major blocks.
+                        */
+                       if (!card_data)
+                               continue;
+               } else {
 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
-                               cards = sgen_card_table_get_card_scan_address ((mword)block_start);
+                       card_data = card_base = sgen_card_table_get_card_scan_address ((mword)block_start);
 #else
-                               cards = cards_data;
-                               if (!sgen_card_table_get_card_data (cards_data, (mword)block_start, CARDS_PER_BLOCK))
-                                       continue;
+                       if (!sgen_card_table_get_card_data (cards_copy, (mword)block_start, CARDS_PER_BLOCK))
+                               continue;
+                       card_data = card_base = cards_copy;
 #endif
-                       }
+               }
+               card_data_end = card_data + CARDS_PER_BLOCK;
 
-                       obj = (char*)MS_BLOCK_OBJ_FAST (block_start, block_obj_size, 0);
-                       end = block_start + MS_BLOCK_SIZE;
-                       base = sgen_card_table_align_pointer (obj);
+               card_data += MS_BLOCK_SKIP >> CARD_BITS;
 
-                       cards += MS_BLOCK_SKIP >> CARD_BITS;
+               card_data = initial_skip_card (card_data);
+               while (card_data < card_data_end) {
+                       size_t card_index, first_object_index;
+                       char *start;
+                       char *end;
+                       char *first_obj, *obj;
 
-                       while (obj < end) {
-                               size_t card_offset;
+                       HEAVY_STAT (++scanned_cards);
 
-                               if (!block->swept)
-                                       sweep_block (block, FALSE);
+                       if (!*card_data) {
+                               ++card_data;
+                               continue;
+                       }
 
-                               if (!MS_OBJ_ALLOCED_FAST (obj, block_start))
-                                       goto next_large;
+                       card_index = card_data - card_base;
+                       start = (char*)(block_start + card_index * CARD_SIZE_IN_BYTES);
+                       end = start + CARD_SIZE_IN_BYTES;
 
-                               if (mod_union) {
-                                       /* FIXME: do this more efficiently */
-                                       int w, b;
-                                       MS_CALC_MARK_BIT (w, b, obj);
-                                       if (!MS_MARK_BIT (block, w, b))
-                                               goto next_large;
-                               }
+                       if (!block->swept)
+                               sweep_block (block, FALSE);
 
-                               card_offset = (obj - base) >> CARD_BITS;
-                               sgen_cardtable_scan_object (obj, block_obj_size, cards + card_offset, mod_union, queue);
+                       HEAVY_STAT (++marked_cards);
 
-                       next_large:
-                               obj += block_obj_size;
-                       }
-               } else {
-                       guint8 *card_data, *card_base;
-                       guint8 *card_data_end;
+                       if (small_objects)
+                               sgen_card_table_prepare_card_for_scanning (card_data);
 
                        /*
-                        * This is safe in face of card aliasing for the following reason:
-                        *
-                        * Major blocks are 16k aligned, or 32 cards aligned.
-                        * Cards aliasing happens in powers of two, so as long as major blocks are aligned to their
-                        * sizes, they won't overflow the cardtable overlap modulus.
+                        * If the card we're looking at starts at or in the block header, we
+                        * must start at the first object in the block, without calculating
+                        * the index of the object we're hypothetically starting at, because
+                        * it would be negative.
                         */
-                       if (mod_union) {
-#ifdef SGEN_HAVE_CONCURRENT_MARK
-                               card_data = card_base = block->cardtable_mod_union;
-                               /*
-                                * This happens when the nursery
-                                * collection that precedes finishing
-                                * the concurrent collection allocates
-                                * new major blocks.
-                                */
-                               if (!card_data)
-                                       continue;
-#else
-                               g_assert_not_reached ();
-                               card_data = NULL;
-#endif
-                       } else {
-                               card_data = card_base = sgen_card_table_get_card_scan_address ((mword)block_start);
-                       }
-                       card_data_end = card_data + CARDS_PER_BLOCK;
-
-                       card_data += MS_BLOCK_SKIP >> CARD_BITS;
-
-                       for (card_data = initial_skip_card (card_data); card_data < card_data_end; ++card_data) { //card_data = skip_card (card_data + 1, card_data_end)) {
-                               size_t index;
-                               size_t idx = card_data - card_base;
-                               char *start = (char*)(block_start + idx * CARD_SIZE_IN_BYTES);
-                               char *end = start + CARD_SIZE_IN_BYTES;
-                               char *first_obj, *obj;
-
-                               HEAVY_STAT (++scanned_cards);
-
-                               if (!*card_data)
-                                       continue;
-
-                               if (!block->swept)
-                                       sweep_block (block, FALSE);
-
-                               HEAVY_STAT (++marked_cards);
-
-                               sgen_card_table_prepare_card_for_scanning (card_data);
+                       if (card_index <= (MS_BLOCK_SKIP >> CARD_BITS))
+                               first_object_index = 0;
+                       else
+                               first_object_index = MS_BLOCK_OBJ_INDEX_FAST (start, block_start, block_obj_size);
 
-                               if (idx == 0)
-                                       index = 0;
-                               else
-                                       index = MS_BLOCK_OBJ_INDEX_FAST (start, block_start, block_obj_size);
+                       obj = first_obj = (char*)MS_BLOCK_OBJ_FAST (block_start, block_obj_size, first_object_index);
 
-                               obj = first_obj = (char*)MS_BLOCK_OBJ_FAST (block_start, block_obj_size, index);
-                               while (obj < end) {
-                                       if (!MS_OBJ_ALLOCED_FAST (obj, block_start))
-                                               goto next_small;
+                       while (obj < end) {
+                               if (obj < scan_front || !MS_OBJ_ALLOCED_FAST (obj, block_start))
+                                       goto next_object;
 
-                                       if (mod_union) {
-                                               /* FIXME: do this more efficiently */
-                                               int w, b;
-                                               MS_CALC_MARK_BIT (w, b, obj);
-                                               if (!MS_MARK_BIT (block, w, b))
-                                                       goto next_small;
-                                       }
+                               if (mod_union) {
+                                       /* FIXME: do this more efficiently */
+                                       int w, b;
+                                       MS_CALC_MARK_BIT (w, b, obj);
+                                       if (!MS_MARK_BIT (block, w, b))
+                                               goto next_object;
+                               }
 
+                               if (small_objects) {
                                        HEAVY_STAT (++scanned_objects);
                                        scan_func (obj, sgen_obj_get_descriptor (obj), queue);
-                               next_small:
-                                       obj += block_obj_size;
+                               } else {
+                                       size_t offset = card_offset (obj, block_start);
+                                       sgen_cardtable_scan_object (obj, block_obj_size, card_base + offset, mod_union, queue);
                                }
-                               HEAVY_STAT (if (*card_data) ++remarked_cards);
-                               binary_protocol_card_scan (first_obj, obj - first_obj);
+                       next_object:
+                               obj += block_obj_size;
+                               g_assert (scan_front <= obj);
+                               scan_front = obj;
                        }
+
+                       HEAVY_STAT (if (*card_data) ++remarked_cards);
+                       binary_protocol_card_scan (first_obj, obj - first_obj);
+
+                       if (small_objects)
+                               ++card_data;
+                       else
+                               card_data = card_base + card_offset (obj, block_start);
                }
        } END_FOREACH_BLOCK;
 }
@@ -1888,7 +1873,6 @@ major_count_cards (long long *num_total_cards, long long *num_marked_cards)
        *num_marked_cards = marked_cards;
 }
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 static void
 update_cardtable_mod_union (void)
 {
@@ -1908,9 +1892,9 @@ static guint8*
 major_get_cardtable_mod_union_for_object (char *obj)
 {
        MSBlockInfo *block = MS_BLOCK_FOR_OBJ (obj);
-       return &block->cardtable_mod_union [(obj - (char*)sgen_card_table_align_pointer (MS_BLOCK_FOR_BLOCK_INFO (block))) >> CARD_BITS];
+       size_t offset = card_offset (obj, (char*)sgen_card_table_align_pointer (MS_BLOCK_FOR_BLOCK_INFO (block)));
+       return &block->cardtable_mod_union [offset];
 }
-#endif
 
 static void
 alloc_free_block_lists (MSBlockInfo ***lists)
@@ -1928,12 +1912,8 @@ post_param_init (SgenMajorCollector *collector)
        collector->sweeps_lazily = lazy_sweep;
 }
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 static void
 sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurrent)
-#else // SGEN_HAVE_CONCURRENT_MARK
-#error unknown configuration
-#endif // SGEN_HAVE_CONCURRENT_MARK
 {
        int i;
 
@@ -1976,22 +1956,17 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 
        collector->section_size = MAJOR_SECTION_SIZE;
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        concurrent_mark = is_concurrent;
        if (is_concurrent) {
                collector->is_concurrent = TRUE;
                collector->want_synchronous_collection = &want_evacuation;
-       } else
-#endif
-       {
+       } else {
                collector->is_concurrent = FALSE;
                collector->want_synchronous_collection = NULL;
        }
        collector->get_and_reset_num_major_objects_marked = major_get_and_reset_num_major_objects_marked;
        collector->supports_cardtable = TRUE;
 
-       collector->have_swept = &have_swept;
-
        collector->alloc_heap = major_alloc_heap;
        collector->is_object_live = major_is_object_live;
        collector->alloc_small_pinned_obj = major_alloc_small_pinned_obj;
@@ -2001,19 +1976,18 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
        collector->free_pinned_object = free_pinned_object;
        collector->iterate_objects = major_iterate_objects;
        collector->free_non_pinned_object = major_free_non_pinned_object;
-       collector->find_pin_queue_start_ends = major_find_pin_queue_start_ends;
        collector->pin_objects = major_pin_objects;
        collector->pin_major_object = pin_major_object;
        collector->scan_card_table = major_scan_card_table;
        collector->iterate_live_block_ranges = (void*)(void*) major_iterate_live_block_ranges;
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        if (is_concurrent) {
                collector->update_cardtable_mod_union = update_cardtable_mod_union;
                collector->get_cardtable_mod_union_for_object = major_get_cardtable_mod_union_for_object;
        }
-#endif
        collector->init_to_space = major_init_to_space;
        collector->sweep = major_sweep;
+       collector->have_finished_sweeping = major_have_finished_sweeping;
+       collector->free_swept_blocks = major_free_swept_blocks;
        collector->check_scan_starts = major_check_scan_starts;
        collector->dump_heap = major_dump_heap;
        collector->get_used_size = major_get_used_size;
@@ -2021,7 +1995,6 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
        collector->finish_nursery_collection = major_finish_nursery_collection;
        collector->start_major_collection = major_start_major_collection;
        collector->finish_major_collection = major_finish_major_collection;
-       collector->have_computed_minor_collection_allowance = major_have_computer_minor_collection_allowance;
        collector->ptr_is_in_non_pinned_space = major_ptr_is_in_non_pinned_space;
        collector->obj_is_from_pinned_alloc = obj_is_from_pinned_alloc;
        collector->report_pinned_memory_usage = major_report_pinned_memory_usage;
@@ -2035,13 +2008,11 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 
        collector->major_ops.copy_or_mark_object = major_copy_or_mark_object_canonical;
        collector->major_ops.scan_object = major_scan_object_with_evacuation;
-#ifdef SGEN_HAVE_CONCURRENT_MARK
        if (is_concurrent) {
                collector->major_concurrent_ops.copy_or_mark_object = major_copy_or_mark_object_concurrent_canonical;
                collector->major_concurrent_ops.scan_object = major_scan_object_no_mark_concurrent;
                collector->major_concurrent_ops.scan_vtype = major_scan_vtype_concurrent;
        }
-#endif
 
 #if !defined (FIXED_HEAP) && !defined (SGEN_PARALLEL_MARK)
        /* FIXME: this will not work with evacuation or the split nursery. */
@@ -2076,7 +2047,6 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
        g_assert ((MS_BLOCK_SIZE % (8 * CARD_SIZE_IN_BYTES)) == 0);
 }
 
-#ifdef SGEN_HAVE_CONCURRENT_MARK
 void
 sgen_marksweep_init (SgenMajorCollector *collector)
 {
@@ -2088,6 +2058,5 @@ sgen_marksweep_conc_init (SgenMajorCollector *collector)
 {
        sgen_marksweep_init_internal (collector, TRUE);
 }
-#endif
 
 #endif