Merge pull request #2869 from BrzVlad/feature-mod-union-opt

[sgen] Optimize mod union usage

diff --combined mono/sgen/sgen-marksweep.c
index d2e3ccdd0093dcc9e68432ef1e8f20b9198651a2,2bbfaff19c66074962f0f2389b190a44d040ff80..4c63191ad447d0ea5c9fb76bf44f9eba3ee19b46
--- 1/mono/sgen/sgen-marksweep.c
--- 2/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@@ -162,7 -162,7 +162,7 @@@ static int fast_block_obj_size_indexes 
  static gboolean *evacuate_block_obj_sizes;
  static float evacuation_threshold = 0.666f;
  
 -static gboolean lazy_sweep = FALSE;
 +static gboolean lazy_sweep = TRUE;
  
  enum {
        SWEEP_STATE_SWEPT,
@@@ -375,11 -375,14 +375,14 @@@ ms_get_empty_block (void
   * list, where it will either be freed later on, or reused in nursery collections.
   */
  static void
- ms_free_block (void *block)
+ ms_free_block (MSBlockInfo *info)
  {
        void *empty;
+       char *block = MS_BLOCK_FOR_BLOCK_INFO (info);
  
        sgen_memgov_release_space (MS_BLOCK_SIZE, SPACE_MAJOR);
+       if (info->cardtable_mod_union)
+               sgen_card_table_free_mod_union (info->cardtable_mod_union, block, MS_BLOCK_SIZE);
        memset (block, 0, MS_BLOCK_SIZE);
  
        do {
@@@ -807,7 -810,6 +810,7 @@@ set_sweep_state (int new_, int expected
  static gboolean ensure_block_is_checked_for_sweeping (guint32 block_index, gboolean wait, gboolean *have_checked);
  
  static SgenThreadPoolJob * volatile sweep_job;
 +static SgenThreadPoolJob * volatile sweep_blocks_job;
  
  static void
  major_finish_sweep_checking (void)
@@@ -1507,10 -1509,8 +1510,8 @@@ ensure_block_is_checked_for_sweeping (g
  
        count = MS_BLOCK_FREE / block->obj_size;
  
-       if (block->cardtable_mod_union) {
-               sgen_card_table_free_mod_union (block->cardtable_mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE);
-               block->cardtable_mod_union = NULL;
-       }
+       if (block->cardtable_mod_union)
+               memset (block->cardtable_mod_union, 0, CARDS_PER_BLOCK);
  
        /* Count marked objects in the block */
        for (i = 0; i < MS_NUM_MARK_WORDS; ++i)
@@@ -1578,20 -1578,6 +1579,20 @@@
        return !!tagged_block;
  }
  
 +static void
 +sweep_blocks_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
 +{
 +      volatile gpointer *slot;
 +
 +      SGEN_ARRAY_LIST_FOREACH_SLOT (&allocated_blocks, slot) {
 +              sweep_block (BLOCK_UNTAG (*slot));
 +      } SGEN_ARRAY_LIST_END_FOREACH_SLOT;
 +
 +      mono_memory_write_barrier ();
 +
 +      sweep_blocks_job = NULL;
 +}
 +
  static void
  sweep_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
  {
@@@ -1633,16 -1619,6 +1634,16 @@@
  
        sgen_array_list_remove_nulls (&allocated_blocks);
  
 +      /*
 +       * Concurrently sweep all the blocks to reduce workload during minor
 +       * pauses where we need certain blocks to be swept. At the start of
 +       * the next major we need all blocks to be swept anyway.
 +       */
 +      if (concurrent_sweep && lazy_sweep) {
 +              sweep_blocks_job = sgen_thread_pool_job_alloc ("sweep_blocks", sweep_blocks_job_func, sizeof (SgenThreadPoolJob));
 +              sgen_thread_pool_job_enqueue (sweep_blocks_job);
 +      }
 +
        sweep_finish ();
  
        sweep_job = NULL;
@@@ -1669,8 -1645,6 +1670,8 @@@ sweep_finish (void
        sgen_memgov_major_post_sweep ();
  
        set_sweep_state (SWEEP_STATE_SWEPT, SWEEP_STATE_COMPACTING);
 +      if (concurrent_sweep)
 +              binary_protocol_concurrent_sweep_end (sgen_timestamp ());
  }
  
  static void
@@@ -1890,22 -1864,12 +1891,22 @@@ major_start_major_collection (void
                sgen_evacuation_freelist_blocks (&free_block_lists [MS_BLOCK_FLAG_REFS][i], i);
        }
  
 -      if (lazy_sweep)
 -              binary_protocol_sweep_begin (GENERATION_OLD, TRUE);
 +      if (lazy_sweep && concurrent_sweep) {
 +              /*
 +               * sweep_blocks_job is created before sweep_finish, which we wait for above
 +               * (major_finish_sweep_checking). After the end of sweep, if we don't have
 +               * sweep_blocks_job set, it means that it has already been run.
 +               */
 +              SgenThreadPoolJob *job = sweep_blocks_job;
 +              if (job)
 +                      sgen_thread_pool_job_wait (job);
 +      }
  
 +      if (lazy_sweep && !concurrent_sweep)
 +              binary_protocol_sweep_begin (GENERATION_OLD, TRUE);
        /* Sweep all unswept blocks and set them to MARKING */
        FOREACH_BLOCK_NO_LOCK (block) {
 -              if (lazy_sweep)
 +              if (lazy_sweep && !concurrent_sweep)
                        sweep_block (block);
                SGEN_ASSERT (0, block->state == BLOCK_STATE_SWEPT, "All blocks must be swept when we're pinning.");
                set_block_state (block, BLOCK_STATE_MARKING, BLOCK_STATE_SWEPT);
@@@ -1917,7 -1881,8 +1918,7 @@@
                if (evacuate_block_obj_sizes [block->obj_size_index] && !block->free_list)
                        block->is_to_space = TRUE;
        } END_FOREACH_BLOCK_NO_LOCK;
 -
 -      if (lazy_sweep)
 +      if (lazy_sweep && !concurrent_sweep)
                binary_protocol_sweep_end (GENERATION_OLD, TRUE);
  
        set_sweep_state (SWEEP_STATE_NEED_SWEEPING, SWEEP_STATE_SWEPT);
@@@ -2484,10 -2449,21 +2485,21 @@@ update_cardtable_mod_union (void
        MSBlockInfo *block;
  
        FOREACH_BLOCK_NO_LOCK (block) {
-               size_t num_cards;
-               guint8 *mod_union = get_cardtable_mod_union_for_block (block, TRUE);
-               sgen_card_table_update_mod_union (mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE, &num_cards);
-               SGEN_ASSERT (6, num_cards == CARDS_PER_BLOCK, "Number of cards calculation is wrong");
+               gpointer *card_start = (gpointer*) sgen_card_table_get_card_address ((mword)MS_BLOCK_FOR_BLOCK_INFO (block));
+               gboolean has_dirty_cards = FALSE;
+               int i;
+               for (i = 0; i < CARDS_PER_BLOCK / sizeof(gpointer); i++) {
+                       if (card_start [i]) {
+                               has_dirty_cards = TRUE;
+                               break;
+                       }
+               }
+               if (has_dirty_cards) {
+                       size_t num_cards;
+                       guint8 *mod_union = get_cardtable_mod_union_for_block (block, TRUE);
+                       sgen_card_table_update_mod_union (mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE, &num_cards);
+                       SGEN_ASSERT (6, num_cards == CARDS_PER_BLOCK, "Number of cards calculation is wrong");
+               }
        } END_FOREACH_BLOCK_NO_LOCK;
  }