mono/metadata/sgen-gc.c

   1 /*
   2  * sgen-gc.c: Simple generational GC.
   3  *
   4  * Author:
   5  *      Paolo Molaro (lupus@ximian.com)
   6  *  Rodrigo Kumpera (kumpera@gmail.com)
   7  *
   8  * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
   9  * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
  10  *
  11  * Thread start/stop adapted from Boehm's GC:
  12  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  13  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  14  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  15  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  16  * Copyright 2001-2003 Ximian, Inc
  17  * Copyright 2003-2010 Novell, Inc.
  18  * Copyright 2011 Xamarin, Inc.
  19  * Copyright (C) 2012 Xamarin Inc
  20  *
  21  * This library is free software; you can redistribute it and/or
  22  * modify it under the terms of the GNU Library General Public
  23  * License 2.0 as published by the Free Software Foundation;
  24  *
  25  * This library is distributed in the hope that it will be useful,
  26  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  28  * Library General Public License for more details.
  29  *
  30  * You should have received a copy of the GNU Library General Public
  31  * License 2.0 along with this library; if not, write to the Free
  32  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  33  *
  34  * Important: allocation provides always zeroed memory, having to do
  35  * a memset after allocation is deadly for performance.
  36  * Memory usage at startup is currently as follows:
  37  * 64 KB pinned space
  38  * 64 KB internal space
  39  * size of nursery
  40  * We should provide a small memory config with half the sizes
  41  *
  42  * We currently try to make as few mono assumptions as possible:
  43  * 1) 2-word header with no GC pointers in it (first vtable, second to store the
  44  *    forwarding ptr)
  45  * 2) gc descriptor is the second word in the vtable (first word in the class)
  46  * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
  47  * 4) there is a function to get an object's size and the number of
  48  *    elements in an array.
  49  * 5) we know the special way bounds are allocated for complex arrays
  50  * 6) we know about proxies and how to treat them when domains are unloaded
  51  *
  52  * Always try to keep stack usage to a minimum: no recursive behaviour
  53  * and no large stack allocs.
  54  *
  55  * General description.
  56  * Objects are initially allocated in a nursery using a fast bump-pointer technique.
  57  * When the nursery is full we start a nursery collection: this is performed with a
  58  * copying GC.
  59  * When the old generation is full we start a copying GC of the old generation as well:
  60  * this will be changed to mark&sweep with copying when fragmentation becomes to severe
  61  * in the future.  Maybe we'll even do both during the same collection like IMMIX.
  62  *
  63  * The things that complicate this description are:
  64  * *) pinned objects: we can't move them so we need to keep track of them
  65  * *) no precise info of the thread stacks and registers: we need to be able to
  66  *    quickly find the objects that may be referenced conservatively and pin them
  67  *    (this makes the first issues more important)
  68  * *) large objects are too expensive to be dealt with using copying GC: we handle them
  69  *    with mark/sweep during major collections
  70  * *) some objects need to not move even if they are small (interned strings, Type handles):
  71  *    we use mark/sweep for them, too: they are not allocated in the nursery, but inside
  72  *    PinnedChunks regions
  73  */
  74
  75 /*
  76  * TODO:
  77
  78  *) we could have a function pointer in MonoClass to implement
  79   customized write barriers for value types
  80
  81  *) investigate the stuff needed to advance a thread to a GC-safe
  82   point (single-stepping, read from unmapped memory etc) and implement it.
  83   This would enable us to inline allocations and write barriers, for example,
  84   or at least parts of them, like the write barrier checks.
  85   We may need this also for handling precise info on stacks, even simple things
  86   as having uninitialized data on the stack and having to wait for the prolog
  87   to zero it. Not an issue for the last frame that we scan conservatively.
  88   We could always not trust the value in the slots anyway.
  89
  90  *) modify the jit to save info about references in stack locations:
  91   this can be done just for locals as a start, so that at least
  92   part of the stack is handled precisely.
  93
  94  *) test/fix endianess issues
  95
  96  *) Implement a card table as the write barrier instead of remembered
  97     sets?  Card tables are not easy to implement with our current
  98     memory layout.  We have several different kinds of major heap
  99     objects: Small objects in regular blocks, small objects in pinned
 100     chunks and LOS objects.  If we just have a pointer we have no way
 101     to tell which kind of object it points into, therefore we cannot
 102     know where its card table is.  The least we have to do to make
 103     this happen is to get rid of write barriers for indirect stores.
 104     (See next item)
 105
 106  *) Get rid of write barriers for indirect stores.  We can do this by
 107     telling the GC to wbarrier-register an object once we do an ldloca
 108     or ldelema on it, and to unregister it once it's not used anymore
 109     (it can only travel downwards on the stack).  The problem with
 110     unregistering is that it needs to happen eventually no matter
 111     what, even if exceptions are thrown, the thread aborts, etc.
 112     Rodrigo suggested that we could do only the registering part and
 113     let the collector find out (pessimistically) when it's safe to
 114     unregister, namely when the stack pointer of the thread that
 115     registered the object is higher than it was when the registering
 116     happened.  This might make for a good first implementation to get
 117     some data on performance.
 118
 119  *) Some sort of blacklist support?  Blacklists is a concept from the
 120     Boehm GC: if during a conservative scan we find pointers to an
 121     area which we might use as heap, we mark that area as unusable, so
 122     pointer retention by random pinning pointers is reduced.
 123
 124  *) experiment with max small object size (very small right now - 2kb,
 125     because it's tied to the max freelist size)
 126
 127   *) add an option to mmap the whole heap in one chunk: it makes for many
 128      simplifications in the checks (put the nursery at the top and just use a single
 129      check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
 130      not flexible (too much of the address space may be used by default or we can't
 131      increase the heap as needed) and we'd need a race-free mechanism to return memory
 132      back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
 133      was written to, munmap is needed, but the following mmap may not find the same segment
 134      free...)
 135
 136  *) memzero the major fragments after restarting the world and optionally a smaller
 137     chunk at a time
 138
 139  *) investigate having fragment zeroing threads
 140
 141  *) separate locks for finalization and other minor stuff to reduce
 142     lock contention
 143
 144  *) try a different copying order to improve memory locality
 145
 146  *) a thread abort after a store but before the write barrier will
 147     prevent the write barrier from executing
 148
 149  *) specialized dynamically generated markers/copiers
 150
 151  *) Dynamically adjust TLAB size to the number of threads.  If we have
 152     too many threads that do allocation, we might need smaller TLABs,
 153     and we might get better performance with larger TLABs if we only
 154     have a handful of threads.  We could sum up the space left in all
 155     assigned TLABs and if that's more than some percentage of the
 156     nursery size, reduce the TLAB size.
 157
 158  *) Explore placing unreachable objects on unused nursery memory.
 159         Instead of memset'ng a region to zero, place an int[] covering it.
 160         A good place to start is add_nursery_frag. The tricky thing here is
 161         placing those objects atomically outside of a collection.
 162
 163  *) Allocation should use asymmetric Dekker synchronization:
 164         http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
 165         This should help weak consistency archs.
 166  */
 167 #include "config.h"
 168 #ifdef HAVE_SGEN_GC
 169
 170 #ifdef __MACH__
 171 #undef _XOPEN_SOURCE
 172 #define _XOPEN_SOURCE
 173 #define _DARWIN_C_SOURCE
 174 #endif
 175
 176 #ifdef HAVE_UNISTD_H
 177 #include <unistd.h>
 178 #endif
 179 #ifdef HAVE_PTHREAD_H
 180 #include <pthread.h>
 181 #endif
 182 #ifdef HAVE_SEMAPHORE_H
 183 #include <semaphore.h>
 184 #endif
 185 #include <stdio.h>
 186 #include <string.h>
 187 #include <signal.h>
 188 #include <errno.h>
 189 #include <assert.h>
 190
 191 #include "metadata/sgen-gc.h"
 192 #include "metadata/metadata-internals.h"
 193 #include "metadata/class-internals.h"
 194 #include "metadata/gc-internal.h"
 195 #include "metadata/object-internals.h"
 196 #include "metadata/threads.h"
 197 #include "metadata/sgen-cardtable.h"
 198 #include "metadata/sgen-ssb.h"
 199 #include "metadata/sgen-protocol.h"
 200 #include "metadata/sgen-archdep.h"
 201 #include "metadata/sgen-bridge.h"
 202 #include "metadata/sgen-memory-governor.h"
 203 #include "metadata/sgen-hash-table.h"
 204 #include "metadata/mono-gc.h"
 205 #include "metadata/method-builder.h"
 206 #include "metadata/profiler-private.h"
 207 #include "metadata/monitor.h"
 208 #include "metadata/threadpool-internals.h"
 209 #include "metadata/mempool-internals.h"
 210 #include "metadata/marshal.h"
 211 #include "metadata/runtime.h"
 212 #include "metadata/sgen-cardtable.h"
 213 #include "metadata/sgen-pinning.h"
 214 #include "metadata/sgen-workers.h"
 215 #include "utils/mono-mmap.h"
 216 #include "utils/mono-time.h"
 217 #include "utils/mono-semaphore.h"
 218 #include "utils/mono-counters.h"
 219 #include "utils/mono-proclib.h"
 220 #include "utils/mono-memory-model.h"
 221 #include "utils/mono-logger-internal.h"
 222 #include "utils/dtrace.h"
 223
 224 #include <mono/utils/mono-logger-internal.h>
 225 #include <mono/utils/memcheck.h>
 226
 227 #if defined(__MACH__)
 228 #include "utils/mach-support.h"
 229 #endif
 230
 231 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
 232         a = i,
 233
 234 enum {
 235 #include "mono/cil/opcode.def"
 236         CEE_LAST
 237 };
 238
 239 #undef OPDEF
 240
 241 #undef pthread_create
 242 #undef pthread_join
 243 #undef pthread_detach
 244
 245 /*
 246  * ######################################################################
 247  * ########  Types and constants used by the GC.
 248  * ######################################################################
 249  */
 250
 251 /* 0 means not initialized, 1 is initialized, -1 means in progress */
 252 static int gc_initialized = 0;
 253 /* If set, check if we need to do something every X allocations */
 254 gboolean has_per_allocation_action;
 255 /* If set, do a heap check every X allocation */
 256 guint32 verify_before_allocs = 0;
 257 /* If set, do a minor collection before every X allocation */
 258 guint32 collect_before_allocs = 0;
 259 /* If set, do a whole heap check before each collection */
 260 static gboolean whole_heap_check_before_collection = FALSE;
 261 /* If set, do a heap consistency check before each minor collection */
 262 static gboolean consistency_check_at_minor_collection = FALSE;
 263 /* If set, check whether mark bits are consistent after major collections */
 264 static gboolean check_mark_bits_after_major_collection = FALSE;
 265 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
 266 static gboolean check_nursery_objects_pinned = FALSE;
 267 /* If set, do a few checks when the concurrent collector is used */
 268 static gboolean do_concurrent_checks = FALSE;
 269 /* If set, check that there are no references to the domain left at domain unload */
 270 static gboolean xdomain_checks = FALSE;
 271 /* If not null, dump the heap after each collection into this file */
 272 static FILE *heap_dump_file = NULL;
 273 /* If set, mark stacks conservatively, even if precise marking is possible */
 274 static gboolean conservative_stack_mark = FALSE;
 275 /* If set, do a plausibility check on the scan_starts before and after
 276    each collection */
 277 static gboolean do_scan_starts_check = FALSE;
 278 static gboolean nursery_collection_is_parallel = FALSE;
 279 static gboolean disable_minor_collections = FALSE;
 280 static gboolean disable_major_collections = FALSE;
 281 gboolean do_pin_stats = FALSE;
 282 static gboolean do_verify_nursery = FALSE;
 283 static gboolean do_dump_nursery_content = FALSE;
 284
 285 #ifdef HEAVY_STATISTICS
 286 long long stat_objects_alloced_degraded = 0;
 287 long long stat_bytes_alloced_degraded = 0;
 288
 289 long long stat_copy_object_called_nursery = 0;
 290 long long stat_objects_copied_nursery = 0;
 291 long long stat_copy_object_called_major = 0;
 292 long long stat_objects_copied_major = 0;
 293
 294 long long stat_scan_object_called_nursery = 0;
 295 long long stat_scan_object_called_major = 0;
 296
 297 long long stat_slots_allocated_in_vain;
 298
 299 long long stat_nursery_copy_object_failed_from_space = 0;
 300 long long stat_nursery_copy_object_failed_forwarded = 0;
 301 long long stat_nursery_copy_object_failed_pinned = 0;
 302 long long stat_nursery_copy_object_failed_to_space = 0;
 303
 304 static int stat_wbarrier_set_field = 0;
 305 static int stat_wbarrier_set_arrayref = 0;
 306 static int stat_wbarrier_arrayref_copy = 0;
 307 static int stat_wbarrier_generic_store = 0;
 308 static int stat_wbarrier_set_root = 0;
 309 static int stat_wbarrier_value_copy = 0;
 310 static int stat_wbarrier_object_copy = 0;
 311 #endif
 312
 313 int stat_minor_gcs = 0;
 314 int stat_major_gcs = 0;
 315
 316 static long long stat_pinned_objects = 0;
 317
 318 static long long time_minor_pre_collection_fragment_clear = 0;
 319 static long long time_minor_pinning = 0;
 320 static long long time_minor_scan_remsets = 0;
 321 static long long time_minor_scan_pinned = 0;
 322 static long long time_minor_scan_registered_roots = 0;
 323 static long long time_minor_scan_thread_data = 0;
 324 static long long time_minor_finish_gray_stack = 0;
 325 static long long time_minor_fragment_creation = 0;
 326
 327 static long long time_major_pre_collection_fragment_clear = 0;
 328 static long long time_major_pinning = 0;
 329 static long long time_major_scan_pinned = 0;
 330 static long long time_major_scan_registered_roots = 0;
 331 static long long time_major_scan_thread_data = 0;
 332 static long long time_major_scan_alloc_pinned = 0;
 333 static long long time_major_scan_finalized = 0;
 334 static long long time_major_scan_big_objects = 0;
 335 static long long time_major_finish_gray_stack = 0;
 336 static long long time_major_free_bigobjs = 0;
 337 static long long time_major_los_sweep = 0;
 338 static long long time_major_sweep = 0;
 339 static long long time_major_fragment_creation = 0;
 340
 341 int gc_debug_level = 0;
 342 FILE* gc_debug_file;
 343
 344 /*
 345 void
 346 mono_gc_flush_info (void)
 347 {
 348         fflush (gc_debug_file);
 349 }
 350 */
 351
 352 #define TV_DECLARE SGEN_TV_DECLARE
 353 #define TV_GETTIME SGEN_TV_GETTIME
 354 #define TV_ELAPSED SGEN_TV_ELAPSED
 355 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
 356
 357 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
 358
 359 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
 360
 361 #define object_is_forwarded     SGEN_OBJECT_IS_FORWARDED
 362 #define object_is_pinned        SGEN_OBJECT_IS_PINNED
 363 #define pin_object              SGEN_PIN_OBJECT
 364 #define unpin_object            SGEN_UNPIN_OBJECT
 365
 366 #define ptr_in_nursery sgen_ptr_in_nursery
 367
 368 #define LOAD_VTABLE     SGEN_LOAD_VTABLE
 369
 370 static const char*
 371 safe_name (void* obj)
 372 {
 373         MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
 374         return vt->klass->name;
 375 }
 376
 377 #define safe_object_get_size    sgen_safe_object_get_size
 378
 379 const char*
 380 sgen_safe_name (void* obj)
 381 {
 382         return safe_name (obj);
 383 }
 384
 385 /*
 386  * ######################################################################
 387  * ########  Global data.
 388  * ######################################################################
 389  */
 390 LOCK_DECLARE (gc_mutex);
 391 static int gc_disabled = 0;
 392
 393 static gboolean use_cardtable;
 394
 395 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
 396
 397 static mword pagesize = 4096;
 398 int degraded_mode = 0;
 399
 400 static mword bytes_pinned_from_failed_allocation = 0;
 401
 402 GCMemSection *nursery_section = NULL;
 403 static mword lowest_heap_address = ~(mword)0;
 404 static mword highest_heap_address = 0;
 405
 406 LOCK_DECLARE (sgen_interruption_mutex);
 407 static LOCK_DECLARE (pin_queue_mutex);
 408
 409 #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex)
 410 #define UNLOCK_PIN_QUEUE mono_mutex_unlock (&pin_queue_mutex)
 411
 412 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
 413 struct _FinalizeReadyEntry {
 414         FinalizeReadyEntry *next;
 415         void *object;
 416 };
 417
 418 typedef struct _EphemeronLinkNode EphemeronLinkNode;
 419
 420 struct _EphemeronLinkNode {
 421         EphemeronLinkNode *next;
 422         char *array;
 423 };
 424
 425 typedef struct {
 426        void *key;
 427        void *value;
 428 } Ephemeron;
 429
 430 int current_collection_generation = -1;
 431 volatile gboolean concurrent_collection_in_progress = FALSE;
 432
 433 /* objects that are ready to be finalized */
 434 static FinalizeReadyEntry *fin_ready_list = NULL;
 435 static FinalizeReadyEntry *critical_fin_list = NULL;
 436
 437 static EphemeronLinkNode *ephemeron_list;
 438
 439 /* registered roots: the key to the hash is the root start address */
 440 /*
 441  * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
 442  */
 443 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
 444         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 445         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 446         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
 447 };
 448 static mword roots_size = 0; /* amount of memory in the root set */
 449
 450 #define GC_ROOT_NUM 32
 451 typedef struct {
 452         int count;              /* must be the first field */
 453         void *objects [GC_ROOT_NUM];
 454         int root_types [GC_ROOT_NUM];
 455         uintptr_t extra_info [GC_ROOT_NUM];
 456 } GCRootReport;
 457
 458 static void
 459 notify_gc_roots (GCRootReport *report)
 460 {
 461         if (!report->count)
 462                 return;
 463         mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
 464         report->count = 0;
 465 }
 466
 467 static void
 468 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
 469 {
 470         if (report->count == GC_ROOT_NUM)
 471                 notify_gc_roots (report);
 472         report->objects [report->count] = object;
 473         report->root_types [report->count] = rtype;
 474         report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
 475 }
 476
 477 MonoNativeTlsKey thread_info_key;
 478
 479 #ifdef HAVE_KW_THREAD
 480 __thread SgenThreadInfo *sgen_thread_info;
 481 __thread gpointer *store_remset_buffer;
 482 __thread long store_remset_buffer_index;
 483 __thread char *stack_end;
 484 __thread long *store_remset_buffer_index_addr;
 485 #endif
 486
 487 /* The size of a TLAB */
 488 /* The bigger the value, the less often we have to go to the slow path to allocate a new
 489  * one, but the more space is wasted by threads not allocating much memory.
 490  * FIXME: Tune this.
 491  * FIXME: Make this self-tuning for each thread.
 492  */
 493 guint32 tlab_size = (1024 * 4);
 494
 495 #define MAX_SMALL_OBJ_SIZE      SGEN_MAX_SMALL_OBJ_SIZE
 496
 497 /* Functions supplied by the runtime to be called by the GC */
 498 static MonoGCCallbacks gc_callbacks;
 499
 500 #define ALLOC_ALIGN             SGEN_ALLOC_ALIGN
 501 #define ALLOC_ALIGN_BITS        SGEN_ALLOC_ALIGN_BITS
 502
 503 #define ALIGN_UP                SGEN_ALIGN_UP
 504
 505 #define MOVED_OBJECTS_NUM 64
 506 static void *moved_objects [MOVED_OBJECTS_NUM];
 507 static int moved_objects_idx = 0;
 508
 509 /* Vtable of the objects used to fill out nursery fragments before a collection */
 510 static MonoVTable *array_fill_vtable;
 511
 512 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
 513 MonoNativeThreadId main_gc_thread = NULL;
 514 #endif
 515
 516 /*Object was pinned during the current collection*/
 517 static mword objects_pinned;
 518
 519 /*
 520  * ######################################################################
 521  * ########  Macros and function declarations.
 522  * ######################################################################
 523  */
 524
 525 inline static void*
 526 align_pointer (void *ptr)
 527 {
 528         mword p = (mword)ptr;
 529         p += sizeof (gpointer) - 1;
 530         p &= ~ (sizeof (gpointer) - 1);
 531         return (void*)p;
 532 }
 533
 534 typedef SgenGrayQueue GrayQueue;
 535
 536 /* forward declarations */
 537 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
 538 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
 539 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
 540 static void report_finalizer_roots (void);
 541 static void report_registered_roots (void);
 542
 543 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
 544 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx);
 545 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
 546
 547 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
 548
 549
 550 static void init_stats (void);
 551
 552 static int mark_ephemerons_in_range (ScanCopyContext ctx);
 553 static void clear_unreachable_ephemerons (gboolean concurrent_cementing, ScanCopyContext ctx);
 554 static void null_ephemerons_for_domain (MonoDomain *domain);
 555
 556 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
 557
 558 SgenObjectOperations current_object_ops;
 559 SgenMajorCollector major_collector;
 560 SgenMinorCollector sgen_minor_collector;
 561 static GrayQueue gray_queue;
 562 static GrayQueue remember_major_objects_gray_queue;
 563
 564 static SgenRemeberedSet remset;
 565
 566 /* The gray queue to use from the main collection thread. */
 567 #define WORKERS_DISTRIBUTE_GRAY_QUEUE   (&gray_queue)
 568
 569 /*
 570  * The gray queue a worker job must use.  If we're not parallel or
 571  * concurrent, we use the main gray queue.
 572  */
 573 static SgenGrayQueue*
 574 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
 575 {
 576         return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
 577 }
 578
 579 static gboolean have_non_collection_major_object_remembers = FALSE;
 580
 581 gboolean
 582 sgen_remember_major_object_for_concurrent_mark (char *obj)
 583 {
 584         if (!major_collector.is_concurrent)
 585                 return FALSE;
 586
 587         g_assert (current_collection_generation == GENERATION_NURSERY || current_collection_generation == -1);
 588
 589         if (!concurrent_collection_in_progress)
 590                 return FALSE;
 591
 592         GRAY_OBJECT_ENQUEUE (&remember_major_objects_gray_queue, obj);
 593
 594         if (current_collection_generation != GENERATION_NURSERY) {
 595                 /*
 596                  * This happens when the mutator allocates large or
 597                  * pinned objects or when allocating in degraded
 598                  * mode.
 599                  */
 600                 have_non_collection_major_object_remembers = TRUE;
 601         }
 602
 603         return TRUE;
 604 }
 605
 606 static void
 607 gray_queue_redirect (SgenGrayQueue *queue)
 608 {
 609         gboolean wake = FALSE;
 610
 611
 612         for (;;) {
 613                 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
 614                 if (!section)
 615                         break;
 616                 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
 617                 wake = TRUE;
 618         }
 619
 620         if (wake) {
 621                 g_assert (concurrent_collection_in_progress ||
 622                                 (current_collection_generation == GENERATION_OLD && major_collector.is_parallel));
 623                 if (sgen_workers_have_started ()) {
 624                         sgen_workers_wake_up_all ();
 625                 } else {
 626                         if (concurrent_collection_in_progress)
 627                                 g_assert (current_collection_generation == -1);
 628                 }
 629         }
 630 }
 631
 632 static void
 633 redirect_major_object_remembers (void)
 634 {
 635         gray_queue_redirect (&remember_major_objects_gray_queue);
 636         have_non_collection_major_object_remembers = FALSE;
 637 }
 638
 639 static gboolean
 640 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
 641 {
 642         MonoObject *o = (MonoObject*)(obj);
 643         MonoObject *ref = (MonoObject*)*(ptr);
 644         int offset = (char*)(ptr) - (char*)o;
 645
 646         if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
 647                 return TRUE;
 648         if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
 649                 return TRUE;
 650         if (mono_class_has_parent_fast (o->vtable->klass, mono_defaults.real_proxy_class) &&
 651                         offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
 652                 return TRUE;
 653         /* Thread.cached_culture_info */
 654         if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
 655                         !strcmp (ref->vtable->klass->name, "CultureInfo") &&
 656                         !strcmp(o->vtable->klass->name_space, "System") &&
 657                         !strcmp(o->vtable->klass->name, "Object[]"))
 658                 return TRUE;
 659         /*
 660          *  at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
 661          * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
 662          * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
 663          * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
 664          * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
 665          * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
 666          * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
 667          * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
 668          * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
 669          */
 670         if (!strcmp (ref->vtable->klass->name_space, "System") &&
 671                         !strcmp (ref->vtable->klass->name, "Byte[]") &&
 672                         !strcmp (o->vtable->klass->name_space, "System.IO") &&
 673                         !strcmp (o->vtable->klass->name, "MemoryStream"))
 674                 return TRUE;
 675         /* append_job() in threadpool.c */
 676         if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
 677                         !strcmp (ref->vtable->klass->name, "AsyncResult") &&
 678                         !strcmp (o->vtable->klass->name_space, "System") &&
 679                         !strcmp (o->vtable->klass->name, "Object[]") &&
 680                         mono_thread_pool_is_queue_array ((MonoArray*) o))
 681                 return TRUE;
 682         return FALSE;
 683 }
 684
 685 static void
 686 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
 687 {
 688         MonoObject *o = (MonoObject*)(obj);
 689         MonoObject *ref = (MonoObject*)*(ptr);
 690         int offset = (char*)(ptr) - (char*)o;
 691         MonoClass *class;
 692         MonoClassField *field;
 693         char *str;
 694
 695         if (!ref || ref->vtable->domain == domain)
 696                 return;
 697         if (is_xdomain_ref_allowed (ptr, obj, domain))
 698                 return;
 699
 700         field = NULL;
 701         for (class = o->vtable->klass; class; class = class->parent) {
 702                 int i;
 703
 704                 for (i = 0; i < class->field.count; ++i) {
 705                         if (class->fields[i].offset == offset) {
 706                                 field = &class->fields[i];
 707                                 break;
 708                         }
 709                 }
 710                 if (field)
 711                         break;
 712         }
 713
 714         if (ref->vtable->klass == mono_defaults.string_class)
 715                 str = mono_string_to_utf8 ((MonoString*)ref);
 716         else
 717                 str = NULL;
 718         g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s)  -  pointed to by:\n",
 719                         o, o->vtable->klass->name_space, o->vtable->klass->name,
 720                         offset, field ? field->name : "",
 721                         ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
 722         mono_gc_scan_for_specific_ref (o, TRUE);
 723         if (str)
 724                 g_free (str);
 725 }
 726
 727 #undef HANDLE_PTR
 728 #define HANDLE_PTR(ptr,obj)     check_reference_for_xdomain ((ptr), (obj), domain)
 729
 730 static void
 731 scan_object_for_xdomain_refs (char *start, mword size, void *data)
 732 {
 733         MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
 734
 735         #include "sgen-scan-object.h"
 736 }
 737
 738 static gboolean scan_object_for_specific_ref_precise = TRUE;
 739
 740 #undef HANDLE_PTR
 741 #define HANDLE_PTR(ptr,obj) do {                \
 742         if ((MonoObject*)*(ptr) == key) {       \
 743         g_print ("found ref to %p in object %p (%s) at offset %td\n",   \
 744                         key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
 745         }                                                               \
 746         } while (0)
 747
 748 static void
 749 scan_object_for_specific_ref (char *start, MonoObject *key)
 750 {
 751         char *forwarded;
 752
 753         if ((forwarded = SGEN_OBJECT_IS_FORWARDED (start)))
 754                 start = forwarded;
 755
 756         if (scan_object_for_specific_ref_precise) {
 757                 #include "sgen-scan-object.h"
 758         } else {
 759                 mword *words = (mword*)start;
 760                 size_t size = safe_object_get_size ((MonoObject*)start);
 761                 int i;
 762                 for (i = 0; i < size / sizeof (mword); ++i) {
 763                         if (words [i] == (mword)key) {
 764                                 g_print ("found possible ref to %p in object %p (%s) at offset %td\n",
 765                                                 key, start, safe_name (start), i * sizeof (mword));
 766                         }
 767                 }
 768         }
 769 }
 770
 771 void
 772 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
 773 {
 774         while (start < end) {
 775                 size_t size;
 776                 char *obj;
 777
 778                 if (!*(void**)start) {
 779                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
 780                         continue;
 781                 }
 782
 783                 if (allow_flags) {
 784                         if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
 785                                 obj = start;
 786                 } else {
 787                         obj = start;
 788                 }
 789
 790                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
 791
 792                 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable)
 793                         callback (obj, size, data);
 794
 795                 start += size;
 796         }
 797 }
 798
 799 static void
 800 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
 801 {
 802         scan_object_for_specific_ref (obj, key);
 803 }
 804
 805 static void
 806 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
 807 {
 808         if (key != obj)
 809                 return;
 810         g_print ("found ref to %p in root record %p\n", key, root);
 811 }
 812
 813 static MonoObject *check_key = NULL;
 814 static RootRecord *check_root = NULL;
 815
 816 static void
 817 check_root_obj_specific_ref_from_marker (void **obj)
 818 {
 819         check_root_obj_specific_ref (check_root, check_key, *obj);
 820 }
 821
 822 static void
 823 scan_roots_for_specific_ref (MonoObject *key, int root_type)
 824 {
 825         void **start_root;
 826         RootRecord *root;
 827         check_key = key;
 828
 829         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 830                 mword desc = root->root_desc;
 831
 832                 check_root = root;
 833
 834                 switch (desc & ROOT_DESC_TYPE_MASK) {
 835                 case ROOT_DESC_BITMAP:
 836                         desc >>= ROOT_DESC_TYPE_SHIFT;
 837                         while (desc) {
 838                                 if (desc & 1)
 839                                         check_root_obj_specific_ref (root, key, *start_root);
 840                                 desc >>= 1;
 841                                 start_root++;
 842                         }
 843                         return;
 844                 case ROOT_DESC_COMPLEX: {
 845                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 846                         int bwords = (*bitmap_data) - 1;
 847                         void **start_run = start_root;
 848                         bitmap_data++;
 849                         while (bwords-- > 0) {
 850                                 gsize bmap = *bitmap_data++;
 851                                 void **objptr = start_run;
 852                                 while (bmap) {
 853                                         if (bmap & 1)
 854                                                 check_root_obj_specific_ref (root, key, *objptr);
 855                                         bmap >>= 1;
 856                                         ++objptr;
 857                                 }
 858                                 start_run += GC_BITS_PER_WORD;
 859                         }
 860                         break;
 861                 }
 862                 case ROOT_DESC_USER: {
 863                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 864                         marker (start_root, check_root_obj_specific_ref_from_marker);
 865                         break;
 866                 }
 867                 case ROOT_DESC_RUN_LEN:
 868                         g_assert_not_reached ();
 869                 default:
 870                         g_assert_not_reached ();
 871                 }
 872         } SGEN_HASH_TABLE_FOREACH_END;
 873
 874         check_key = NULL;
 875         check_root = NULL;
 876 }
 877
 878 void
 879 mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise)
 880 {
 881         void **ptr;
 882         RootRecord *root;
 883
 884         scan_object_for_specific_ref_precise = precise;
 885
 886         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 887                         (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key, TRUE);
 888
 889         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 890
 891         sgen_los_iterate_objects ((IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 892
 893         scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
 894         scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
 895
 896         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], ptr, root) {
 897                 while (ptr < (void**)root->end_root) {
 898                         check_root_obj_specific_ref (root, *ptr, key);
 899                         ++ptr;
 900                 }
 901         } SGEN_HASH_TABLE_FOREACH_END;
 902 }
 903
 904 static gboolean
 905 need_remove_object_for_domain (char *start, MonoDomain *domain)
 906 {
 907         if (mono_object_domain (start) == domain) {
 908                 SGEN_LOG (4, "Need to cleanup object %p", start);
 909                 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
 910                 return TRUE;
 911         }
 912         return FALSE;
 913 }
 914
 915 static void
 916 process_object_for_domain_clearing (char *start, MonoDomain *domain)
 917 {
 918         GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
 919         if (vt->klass == mono_defaults.internal_thread_class)
 920                 g_assert (mono_object_domain (start) == mono_get_root_domain ());
 921         /* The object could be a proxy for an object in the domain
 922            we're deleting. */
 923         if (mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
 924                 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
 925
 926                 /* The server could already have been zeroed out, so
 927                    we need to check for that, too. */
 928                 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
 929                         SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
 930                         ((MonoRealProxy*)start)->unwrapped_server = NULL;
 931                 }
 932         }
 933 }
 934
 935 static MonoDomain *check_domain = NULL;
 936
 937 static void
 938 check_obj_not_in_domain (void **o)
 939 {
 940         g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
 941 }
 942
 943 static void
 944 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
 945 {
 946         void **start_root;
 947         RootRecord *root;
 948         check_domain = domain;
 949         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 950                 mword desc = root->root_desc;
 951
 952                 /* The MonoDomain struct is allowed to hold
 953                    references to objects in its own domain. */
 954                 if (start_root == (void**)domain)
 955                         continue;
 956
 957                 switch (desc & ROOT_DESC_TYPE_MASK) {
 958                 case ROOT_DESC_BITMAP:
 959                         desc >>= ROOT_DESC_TYPE_SHIFT;
 960                         while (desc) {
 961                                 if ((desc & 1) && *start_root)
 962                                         check_obj_not_in_domain (*start_root);
 963                                 desc >>= 1;
 964                                 start_root++;
 965                         }
 966                         break;
 967                 case ROOT_DESC_COMPLEX: {
 968                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 969                         int bwords = (*bitmap_data) - 1;
 970                         void **start_run = start_root;
 971                         bitmap_data++;
 972                         while (bwords-- > 0) {
 973                                 gsize bmap = *bitmap_data++;
 974                                 void **objptr = start_run;
 975                                 while (bmap) {
 976                                         if ((bmap & 1) && *objptr)
 977                                                 check_obj_not_in_domain (*objptr);
 978                                         bmap >>= 1;
 979                                         ++objptr;
 980                                 }
 981                                 start_run += GC_BITS_PER_WORD;
 982                         }
 983                         break;
 984                 }
 985                 case ROOT_DESC_USER: {
 986                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 987                         marker (start_root, check_obj_not_in_domain);
 988                         break;
 989                 }
 990                 case ROOT_DESC_RUN_LEN:
 991                         g_assert_not_reached ();
 992                 default:
 993                         g_assert_not_reached ();
 994                 }
 995         } SGEN_HASH_TABLE_FOREACH_END;
 996
 997         check_domain = NULL;
 998 }
 999
1000 static void
1001 check_for_xdomain_refs (void)
1002 {
1003         LOSObject *bigobj;
1004
1005         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1006                         (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL, FALSE);
1007
1008         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
1009
1010         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1011                 scan_object_for_xdomain_refs (bigobj->data, sgen_los_object_size (bigobj), NULL);
1012 }
1013
1014 static gboolean
1015 clear_domain_process_object (char *obj, MonoDomain *domain)
1016 {
1017         gboolean remove;
1018
1019         process_object_for_domain_clearing (obj, domain);
1020         remove = need_remove_object_for_domain (obj, domain);
1021
1022         if (remove && ((MonoObject*)obj)->synchronisation) {
1023                 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1024                 if (dislink)
1025                         sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
1026         }
1027
1028         return remove;
1029 }
1030
1031 static void
1032 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1033 {
1034         if (clear_domain_process_object (obj, domain))
1035                 memset (obj, 0, size);
1036 }
1037
1038 static void
1039 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1040 {
1041         clear_domain_process_object (obj, domain);
1042 }
1043
1044 static void
1045 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1046 {
1047         if (need_remove_object_for_domain (obj, domain))
1048                 major_collector.free_non_pinned_object (obj, size);
1049 }
1050
1051 static void
1052 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1053 {
1054         if (need_remove_object_for_domain (obj, domain))
1055                 major_collector.free_pinned_object (obj, size);
1056 }
1057
1058 /*
1059  * When appdomains are unloaded we can easily remove objects that have finalizers,
1060  * but all the others could still be present in random places on the heap.
1061  * We need a sweep to get rid of them even though it's going to be costly
1062  * with big heaps.
1063  * The reason we need to remove them is because we access the vtable and class
1064  * structures to know the object size and the reference bitmap: once the domain is
1065  * unloaded the point to random memory.
1066  */
1067 void
1068 mono_gc_clear_domain (MonoDomain * domain)
1069 {
1070         LOSObject *bigobj, *prev;
1071         int i;
1072
1073         LOCK_GC;
1074
1075         if (concurrent_collection_in_progress)
1076                 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
1077         g_assert (!concurrent_collection_in_progress);
1078
1079         sgen_process_fin_stage_entries ();
1080         sgen_process_dislink_stage_entries ();
1081
1082         sgen_clear_nursery_fragments ();
1083
1084         if (xdomain_checks && domain != mono_get_root_domain ()) {
1085                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1086                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1087                 check_for_xdomain_refs ();
1088         }
1089
1090         /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1091         to memory returned to the OS.*/
1092         null_ephemerons_for_domain (domain);
1093
1094         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1095                 sgen_null_links_for_domain (domain, i);
1096
1097         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1098                 sgen_remove_finalizers_for_domain (domain, i);
1099
1100         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1101                         (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
1102
1103         /* We need two passes over major and large objects because
1104            freeing such objects might give their memory back to the OS
1105            (in the case of large objects) or obliterate its vtable
1106            (pinned objects with major-copying or pinned and non-pinned
1107            objects with major-mark&sweep), but we might need to
1108            dereference a pointer from an object to another object if
1109            the first object is a proxy. */
1110         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1111         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1112                 clear_domain_process_object (bigobj->data, domain);
1113
1114         prev = NULL;
1115         for (bigobj = los_object_list; bigobj;) {
1116                 if (need_remove_object_for_domain (bigobj->data, domain)) {
1117                         LOSObject *to_free = bigobj;
1118                         if (prev)
1119                                 prev->next = bigobj->next;
1120                         else
1121                                 los_object_list = bigobj->next;
1122                         bigobj = bigobj->next;
1123                         SGEN_LOG (4, "Freeing large object %p", bigobj->data);
1124                         sgen_los_free_object (to_free);
1125                         continue;
1126                 }
1127                 prev = bigobj;
1128                 bigobj = bigobj->next;
1129         }
1130         major_collector.iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1131         major_collector.iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1132
1133         if (G_UNLIKELY (do_pin_stats)) {
1134                 if (domain == mono_get_root_domain ())
1135                         sgen_pin_stats_print_class_stats ();
1136         }
1137
1138         UNLOCK_GC;
1139 }
1140
1141 /*
1142  * sgen_add_to_global_remset:
1143  *
1144  *   The global remset contains locations which point into newspace after
1145  * a minor collection. This can happen if the objects they point to are pinned.
1146  *
1147  * LOCKING: If called from a parallel collector, the global remset
1148  * lock must be held.  For serial collectors that is not necessary.
1149  */
1150 void
1151 sgen_add_to_global_remset (gpointer ptr, gpointer obj, gboolean concurrent_cementing)
1152 {
1153         SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
1154
1155         if (!major_collector.is_concurrent) {
1156                 SGEN_ASSERT (5, !concurrent_cementing, "Concurrent cementing must only happen with the concurrent collector");
1157                 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
1158         } else {
1159                 if (current_collection_generation == -1)
1160                         SGEN_ASSERT (5, concurrent_cementing, "Global remsets outside of collection pauses can only be added by the concurrent collector");
1161                 if (concurrent_cementing)
1162                         SGEN_ASSERT (5, concurrent_collection_in_progress, "Concurrent collection must be in process in order to add global remsets");
1163         }
1164
1165         if (!object_is_pinned (obj))
1166                 SGEN_ASSERT (5, concurrent_cementing || sgen_minor_collector.is_split, "Non-pinned objects can only remain in nursery if it is a split nursery");
1167         else if (sgen_cement_lookup_or_register (obj, concurrent_cementing))
1168                 return;
1169
1170         remset.record_pointer (ptr);
1171
1172 #ifdef ENABLE_DTRACE
1173         if (G_UNLIKELY (do_pin_stats))
1174                 sgen_pin_stats_register_global_remset (obj);
1175
1176         SGEN_LOG (8, "Adding global remset for %p", ptr);
1177         binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
1178
1179         HEAVY_STAT (++stat_global_remsets_added);
1180
1181         if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
1182                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
1183                 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
1184                                 vt->klass->name_space, vt->klass->name);
1185         }
1186 #endif
1187 }
1188
1189 /*
1190  * sgen_drain_gray_stack:
1191  *
1192  *   Scan objects in the gray stack until the stack is empty. This should be called
1193  * frequently after each object is copied, to achieve better locality and cache
1194  * usage.
1195  */
1196 gboolean
1197 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
1198 {
1199         char *obj;
1200         ScanObjectFunc scan_func = ctx.scan_func;
1201         GrayQueue *queue = ctx.queue;
1202
1203         if (max_objs == -1) {
1204                 for (;;) {
1205                         GRAY_OBJECT_DEQUEUE (queue, obj);
1206                         if (!obj)
1207                                 return TRUE;
1208                         SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1209                         scan_func (obj, queue);
1210                 }
1211         } else {
1212                 int i;
1213
1214                 do {
1215                         for (i = 0; i != max_objs; ++i) {
1216                                 GRAY_OBJECT_DEQUEUE (queue, obj);
1217                                 if (!obj)
1218                                         return TRUE;
1219                                 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1220                                 scan_func (obj, queue);
1221                         }
1222                 } while (max_objs < 0);
1223                 return FALSE;
1224         }
1225 }
1226
1227 /*
1228  * Addresses from start to end are already sorted. This function finds
1229  * the object header for each address and pins the object. The
1230  * addresses must be inside the passed section.  The (start of the)
1231  * address array is overwritten with the addresses of the actually
1232  * pinned objects.  Return the number of pinned objects.
1233  */
1234 static int
1235 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx)
1236 {
1237         void *last = NULL;
1238         int count = 0;
1239         void *search_start;
1240         void *last_obj = NULL;
1241         size_t last_obj_size = 0;
1242         void *addr;
1243         int idx;
1244         void **definitely_pinned = start;
1245         ScanObjectFunc scan_func = ctx.scan_func;
1246         SgenGrayQueue *queue = ctx.queue;
1247
1248         sgen_nursery_allocator_prepare_for_pinning ();
1249
1250         while (start < end) {
1251                 addr = *start;
1252                 /* the range check should be reduntant */
1253                 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1254                         SGEN_LOG (5, "Considering pinning addr %p", addr);
1255                         /* multiple pointers to the same object */
1256                         if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1257                                 start++;
1258                                 continue;
1259                         }
1260                         idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1261                         g_assert (idx < section->num_scan_start);
1262                         search_start = (void*)section->scan_starts [idx];
1263                         if (!search_start || search_start > addr) {
1264                                 while (idx) {
1265                                         --idx;
1266                                         search_start = section->scan_starts [idx];
1267                                         if (search_start && search_start <= addr)
1268                                                 break;
1269                                 }
1270                                 if (!search_start || search_start > addr)
1271                                         search_start = start_nursery;
1272                         }
1273                         if (search_start < last_obj)
1274                                 search_start = (char*)last_obj + last_obj_size;
1275                         /* now addr should be in an object a short distance from search_start
1276                          * Note that search_start must point to zeroed mem or point to an object.
1277                          */
1278
1279                         do {
1280                                 if (!*(void**)search_start) {
1281                                         /* Consistency check */
1282                                         /*
1283                                         for (frag = nursery_fragments; frag; frag = frag->next) {
1284                                                 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1285                                                         g_assert_not_reached ();
1286                                         }
1287                                         */
1288
1289                                         search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
1290                                         continue;
1291                                 }
1292                                 last_obj = search_start;
1293                                 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1294
1295                                 if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) {
1296                                         /* Marks the beginning of a nursery fragment, skip */
1297                                 } else {
1298                                         SGEN_LOG (8, "Pinned try match %p (%s), size %zd", last_obj, safe_name (last_obj), last_obj_size);
1299                                         if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1300                                                 if (scan_func) {
1301                                                         scan_func (search_start, queue);
1302                                                 } else {
1303                                                         SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1304                                                                         search_start, *(void**)search_start, safe_name (search_start), count);
1305                                                         binary_protocol_pin (search_start,
1306                                                                         (gpointer)LOAD_VTABLE (search_start),
1307                                                                         safe_object_get_size (search_start));
1308
1309 #ifdef ENABLE_DTRACE
1310                                                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1311                                                                 int gen = sgen_ptr_in_nursery (search_start) ? GENERATION_NURSERY : GENERATION_OLD;
1312                                                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (search_start);
1313                                                                 MONO_GC_OBJ_PINNED ((mword)search_start,
1314                                                                                 sgen_safe_object_get_size (search_start),
1315                                                                                 vt->klass->name_space, vt->klass->name, gen);
1316                                                         }
1317 #endif
1318
1319                                                         pin_object (search_start);
1320                                                         GRAY_OBJECT_ENQUEUE (queue, search_start);
1321                                                         if (G_UNLIKELY (do_pin_stats))
1322                                                                 sgen_pin_stats_register_object (search_start, last_obj_size);
1323                                                         definitely_pinned [count] = search_start;
1324                                                         count++;
1325                                                 }
1326                                                 break;
1327                                         }
1328                                 }
1329                                 /* skip to the next object */
1330                                 search_start = (void*)((char*)search_start + last_obj_size);
1331                         } while (search_start <= addr);
1332                         /* we either pinned the correct object or we ignored the addr because
1333                          * it points to unused zeroed memory.
1334                          */
1335                         last = addr;
1336                 }
1337                 start++;
1338         }
1339         //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1340         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1341                 GCRootReport report;
1342                 report.count = 0;
1343                 for (idx = 0; idx < count; ++idx)
1344                         add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1345                 notify_gc_roots (&report);
1346         }
1347         stat_pinned_objects += count;
1348         return count;
1349 }
1350
1351 void
1352 sgen_pin_objects_in_section (GCMemSection *section, ScanCopyContext ctx)
1353 {
1354         int num_entries = section->pin_queue_num_entries;
1355         if (num_entries) {
1356                 void **start = section->pin_queue_start;
1357                 int reduced_to;
1358                 reduced_to = pin_objects_from_addresses (section, start, start + num_entries,
1359                                 section->data, section->next_data, ctx);
1360                 section->pin_queue_num_entries = reduced_to;
1361                 if (!reduced_to)
1362                         section->pin_queue_start = NULL;
1363         }
1364 }
1365
1366
1367 void
1368 sgen_pin_object (void *object, GrayQueue *queue)
1369 {
1370         g_assert (!concurrent_collection_in_progress);
1371
1372         if (sgen_collection_is_parallel ()) {
1373                 LOCK_PIN_QUEUE;
1374                 /*object arrives pinned*/
1375                 sgen_pin_stage_ptr (object);
1376                 ++objects_pinned ;
1377                 UNLOCK_PIN_QUEUE;
1378         } else {
1379                 SGEN_PIN_OBJECT (object);
1380                 sgen_pin_stage_ptr (object);
1381                 ++objects_pinned;
1382                 if (G_UNLIKELY (do_pin_stats))
1383                         sgen_pin_stats_register_object (object, safe_object_get_size (object));
1384         }
1385         GRAY_OBJECT_ENQUEUE (queue, object);
1386         binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1387
1388 #ifdef ENABLE_DTRACE
1389         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1390                 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1391                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1392                 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1393         }
1394 #endif
1395 }
1396
1397 void
1398 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1399 {
1400         for (;;) {
1401                 mword vtable_word;
1402                 gboolean major_pinned = FALSE;
1403
1404                 if (sgen_ptr_in_nursery (obj)) {
1405                         if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
1406                                 sgen_pin_object (obj, queue);
1407                                 break;
1408                         }
1409                 } else {
1410                         major_collector.pin_major_object (obj, queue);
1411                         major_pinned = TRUE;
1412                 }
1413
1414                 vtable_word = *(mword*)obj;
1415                 /*someone else forwarded it, update the pointer and bail out*/
1416                 if (vtable_word & SGEN_FORWARDED_BIT) {
1417                         *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
1418                         break;
1419                 }
1420
1421                 /*someone pinned it, nothing to do.*/
1422                 if (vtable_word & SGEN_PINNED_BIT || major_pinned)
1423                         break;
1424         }
1425 }
1426
1427 /* Sort the addresses in array in increasing order.
1428  * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1429  */
1430 void
1431 sgen_sort_addresses (void **array, int size)
1432 {
1433         int i;
1434         void *tmp;
1435
1436         for (i = 1; i < size; ++i) {
1437                 int child = i;
1438                 while (child > 0) {
1439                         int parent = (child - 1) / 2;
1440
1441                         if (array [parent] >= array [child])
1442                                 break;
1443
1444                         tmp = array [parent];
1445                         array [parent] = array [child];
1446                         array [child] = tmp;
1447
1448                         child = parent;
1449                 }
1450         }
1451
1452         for (i = size - 1; i > 0; --i) {
1453                 int end, root;
1454                 tmp = array [i];
1455                 array [i] = array [0];
1456                 array [0] = tmp;
1457
1458                 end = i - 1;
1459                 root = 0;
1460
1461                 while (root * 2 + 1 <= end) {
1462                         int child = root * 2 + 1;
1463
1464                         if (child < end && array [child] < array [child + 1])
1465                                 ++child;
1466                         if (array [root] >= array [child])
1467                                 break;
1468
1469                         tmp = array [root];
1470                         array [root] = array [child];
1471                         array [child] = tmp;
1472
1473                         root = child;
1474                 }
1475         }
1476 }
1477
1478 /*
1479  * Scan the memory between start and end and queue values which could be pointers
1480  * to the area between start_nursery and end_nursery for later consideration.
1481  * Typically used for thread stacks.
1482  */
1483 static void
1484 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1485 {
1486         int count = 0;
1487
1488 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1489         VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1490 #endif
1491
1492         while (start < end) {
1493                 if (*start >= start_nursery && *start < end_nursery) {
1494                         /*
1495                          * *start can point to the middle of an object
1496                          * note: should we handle pointing at the end of an object?
1497                          * pinning in C# code disallows pointing at the end of an object
1498                          * but there is some small chance that an optimizing C compiler
1499                          * may keep the only reference to an object by pointing
1500                          * at the end of it. We ignore this small chance for now.
1501                          * Pointers to the end of an object are indistinguishable
1502                          * from pointers to the start of the next object in memory
1503                          * so if we allow that we'd need to pin two objects...
1504                          * We queue the pointer in an array, the
1505                          * array will then be sorted and uniqued. This way
1506                          * we can coalesce several pinning pointers and it should
1507                          * be faster since we'd do a memory scan with increasing
1508                          * addresses. Note: we can align the address to the allocation
1509                          * alignment, so the unique process is more effective.
1510                          */
1511                         mword addr = (mword)*start;
1512                         addr &= ~(ALLOC_ALIGN - 1);
1513                         if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1514                                 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1515                                 sgen_pin_stage_ptr ((void*)addr);
1516                                 count++;
1517                         }
1518                         if (G_UNLIKELY (do_pin_stats)) {
1519                                 if (ptr_in_nursery ((void*)addr))
1520                                         sgen_pin_stats_register_address ((char*)addr, pin_type);
1521                         }
1522                 }
1523                 start++;
1524         }
1525         if (count)
1526                 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1527 }
1528
1529 /*
1530  * The first thing we do in a collection is to identify pinned objects.
1531  * This function considers all the areas of memory that need to be
1532  * conservatively scanned.
1533  */
1534 static void
1535 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1536 {
1537         void **start_root;
1538         RootRecord *root;
1539         SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1540         /* objects pinned from the API are inside these roots */
1541         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1542                 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1543                 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1544         } SGEN_HASH_TABLE_FOREACH_END;
1545         /* now deal with the thread stacks
1546          * in the future we should be able to conservatively scan only:
1547          * *) the cpu registers
1548          * *) the unmanaged stack frames
1549          * *) the _last_ managed stack frame
1550          * *) pointers slots in managed frames
1551          */
1552         scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1553 }
1554
1555 static void
1556 unpin_objects_from_queue (SgenGrayQueue *queue)
1557 {
1558         for (;;) {
1559                 char *addr;
1560                 GRAY_OBJECT_DEQUEUE (queue, addr);
1561                 if (!addr)
1562                         break;
1563                 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1564                 SGEN_UNPIN_OBJECT (addr);
1565         }
1566 }
1567
1568 typedef struct {
1569         CopyOrMarkObjectFunc func;
1570         GrayQueue *queue;
1571 } UserCopyOrMarkData;
1572
1573 static MonoNativeTlsKey user_copy_or_mark_key;
1574
1575 static void
1576 init_user_copy_or_mark_key (void)
1577 {
1578         mono_native_tls_alloc (&user_copy_or_mark_key, NULL);
1579 }
1580
1581 static void
1582 set_user_copy_or_mark_data (UserCopyOrMarkData *data)
1583 {
1584         mono_native_tls_set_value (user_copy_or_mark_key, data);
1585 }
1586
1587 static void
1588 single_arg_user_copy_or_mark (void **obj)
1589 {
1590         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
1591
1592         data->func (obj, data->queue);
1593 }
1594
1595 /*
1596  * The memory area from start_root to end_root contains pointers to objects.
1597  * Their position is precisely described by @desc (this means that the pointer
1598  * can be either NULL or the pointer to the start of an object).
1599  * This functions copies them to to_space updates them.
1600  *
1601  * This function is not thread-safe!
1602  */
1603 static void
1604 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1605 {
1606         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1607         SgenGrayQueue *queue = ctx.queue;
1608
1609         switch (desc & ROOT_DESC_TYPE_MASK) {
1610         case ROOT_DESC_BITMAP:
1611                 desc >>= ROOT_DESC_TYPE_SHIFT;
1612                 while (desc) {
1613                         if ((desc & 1) && *start_root) {
1614                                 copy_func (start_root, queue);
1615                                 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1616                                 sgen_drain_gray_stack (-1, ctx);
1617                         }
1618                         desc >>= 1;
1619                         start_root++;
1620                 }
1621                 return;
1622         case ROOT_DESC_COMPLEX: {
1623                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1624                 int bwords = (*bitmap_data) - 1;
1625                 void **start_run = start_root;
1626                 bitmap_data++;
1627                 while (bwords-- > 0) {
1628                         gsize bmap = *bitmap_data++;
1629                         void **objptr = start_run;
1630                         while (bmap) {
1631                                 if ((bmap & 1) && *objptr) {
1632                                         copy_func (objptr, queue);
1633                                         SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1634                                         sgen_drain_gray_stack (-1, ctx);
1635                                 }
1636                                 bmap >>= 1;
1637                                 ++objptr;
1638                         }
1639                         start_run += GC_BITS_PER_WORD;
1640                 }
1641                 break;
1642         }
1643         case ROOT_DESC_USER: {
1644                 UserCopyOrMarkData data = { copy_func, queue };
1645                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1646                 set_user_copy_or_mark_data (&data);
1647                 marker (start_root, single_arg_user_copy_or_mark);
1648                 set_user_copy_or_mark_data (NULL);
1649                 break;
1650         }
1651         case ROOT_DESC_RUN_LEN:
1652                 g_assert_not_reached ();
1653         default:
1654                 g_assert_not_reached ();
1655         }
1656 }
1657
1658 static void
1659 reset_heap_boundaries (void)
1660 {
1661         lowest_heap_address = ~(mword)0;
1662         highest_heap_address = 0;
1663 }
1664
1665 void
1666 sgen_update_heap_boundaries (mword low, mword high)
1667 {
1668         mword old;
1669
1670         do {
1671                 old = lowest_heap_address;
1672                 if (low >= old)
1673                         break;
1674         } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1675
1676         do {
1677                 old = highest_heap_address;
1678                 if (high <= old)
1679                         break;
1680         } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1681 }
1682
1683 /*
1684  * Allocate and setup the data structures needed to be able to allocate objects
1685  * in the nursery. The nursery is stored in nursery_section.
1686  */
1687 static void
1688 alloc_nursery (void)
1689 {
1690         GCMemSection *section;
1691         char *data;
1692         int scan_starts;
1693         int alloc_size;
1694
1695         if (nursery_section)
1696                 return;
1697         SGEN_LOG (2, "Allocating nursery size: %lu", (unsigned long)sgen_nursery_size);
1698         /* later we will alloc a larger area for the nursery but only activate
1699          * what we need. The rest will be used as expansion if we have too many pinned
1700          * objects in the existing nursery.
1701          */
1702         /* FIXME: handle OOM */
1703         section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1704
1705         alloc_size = sgen_nursery_size;
1706
1707         /* If there isn't enough space even for the nursery we should simply abort. */
1708         g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1709
1710 #ifdef SGEN_ALIGN_NURSERY
1711         data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1712 #else
1713         data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1714 #endif
1715         sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1716         SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1717         section->data = section->next_data = data;
1718         section->size = alloc_size;
1719         section->end_data = data + sgen_nursery_size;
1720         scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1721         section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1722         section->num_scan_start = scan_starts;
1723
1724         nursery_section = section;
1725
1726         sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1727 }
1728
1729 void*
1730 mono_gc_get_nursery (int *shift_bits, size_t *size)
1731 {
1732         *size = sgen_nursery_size;
1733 #ifdef SGEN_ALIGN_NURSERY
1734         *shift_bits = DEFAULT_NURSERY_BITS;
1735 #else
1736         *shift_bits = -1;
1737 #endif
1738         return sgen_get_nursery_start ();
1739 }
1740
1741 void
1742 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1743 {
1744         SgenThreadInfo *info = mono_thread_info_current ();
1745
1746         /* Could be called from sgen_thread_unregister () with a NULL info */
1747         if (domain) {
1748                 g_assert (info);
1749                 info->stopped_domain = domain;
1750         }
1751 }
1752
1753 gboolean
1754 mono_gc_precise_stack_mark_enabled (void)
1755 {
1756         return !conservative_stack_mark;
1757 }
1758
1759 FILE *
1760 mono_gc_get_logfile (void)
1761 {
1762         return gc_debug_file;
1763 }
1764
1765 static void
1766 report_finalizer_roots_list (FinalizeReadyEntry *list)
1767 {
1768         GCRootReport report;
1769         FinalizeReadyEntry *fin;
1770
1771         report.count = 0;
1772         for (fin = list; fin; fin = fin->next) {
1773                 if (!fin->object)
1774                         continue;
1775                 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1776         }
1777         notify_gc_roots (&report);
1778 }
1779
1780 static void
1781 report_finalizer_roots (void)
1782 {
1783         report_finalizer_roots_list (fin_ready_list);
1784         report_finalizer_roots_list (critical_fin_list);
1785 }
1786
1787 static GCRootReport *root_report;
1788
1789 static void
1790 single_arg_report_root (void **obj)
1791 {
1792         if (*obj)
1793                 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1794 }
1795
1796 static void
1797 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1798 {
1799         switch (desc & ROOT_DESC_TYPE_MASK) {
1800         case ROOT_DESC_BITMAP:
1801                 desc >>= ROOT_DESC_TYPE_SHIFT;
1802                 while (desc) {
1803                         if ((desc & 1) && *start_root) {
1804                                 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1805                         }
1806                         desc >>= 1;
1807                         start_root++;
1808                 }
1809                 return;
1810         case ROOT_DESC_COMPLEX: {
1811                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1812                 int bwords = (*bitmap_data) - 1;
1813                 void **start_run = start_root;
1814                 bitmap_data++;
1815                 while (bwords-- > 0) {
1816                         gsize bmap = *bitmap_data++;
1817                         void **objptr = start_run;
1818                         while (bmap) {
1819                                 if ((bmap & 1) && *objptr) {
1820                                         add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1821                                 }
1822                                 bmap >>= 1;
1823                                 ++objptr;
1824                         }
1825                         start_run += GC_BITS_PER_WORD;
1826                 }
1827                 break;
1828         }
1829         case ROOT_DESC_USER: {
1830                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1831                 root_report = report;
1832                 marker (start_root, single_arg_report_root);
1833                 break;
1834         }
1835         case ROOT_DESC_RUN_LEN:
1836                 g_assert_not_reached ();
1837         default:
1838                 g_assert_not_reached ();
1839         }
1840 }
1841
1842 static void
1843 report_registered_roots_by_type (int root_type)
1844 {
1845         GCRootReport report;
1846         void **start_root;
1847         RootRecord *root;
1848         report.count = 0;
1849         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1850                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1851                 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1852         } SGEN_HASH_TABLE_FOREACH_END;
1853         notify_gc_roots (&report);
1854 }
1855
1856 static void
1857 report_registered_roots (void)
1858 {
1859         report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1860         report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1861 }
1862
1863 static void
1864 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1865 {
1866         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1867         SgenGrayQueue *queue = ctx.queue;
1868         FinalizeReadyEntry *fin;
1869
1870         for (fin = list; fin; fin = fin->next) {
1871                 if (!fin->object)
1872                         continue;
1873                 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1874                 copy_func (&fin->object, queue);
1875         }
1876 }
1877
1878 static const char*
1879 generation_name (int generation)
1880 {
1881         switch (generation) {
1882         case GENERATION_NURSERY: return "nursery";
1883         case GENERATION_OLD: return "old";
1884         default: g_assert_not_reached ();
1885         }
1886 }
1887
1888 const char*
1889 sgen_generation_name (int generation)
1890 {
1891         return generation_name (generation);
1892 }
1893
1894 SgenObjectOperations *
1895 sgen_get_current_object_ops (void){
1896         return &current_object_ops;
1897 }
1898
1899
1900 static void
1901 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
1902 {
1903         TV_DECLARE (atv);
1904         TV_DECLARE (btv);
1905         int done_with_ephemerons, ephemeron_rounds = 0;
1906         CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1907         ScanObjectFunc scan_func = current_object_ops.scan_object;
1908         ScanCopyContext ctx = { scan_func, copy_func, queue };
1909
1910         /*
1911          * We copied all the reachable objects. Now it's the time to copy
1912          * the objects that were not referenced by the roots, but by the copied objects.
1913          * we built a stack of objects pointed to by gray_start: they are
1914          * additional roots and we may add more items as we go.
1915          * We loop until gray_start == gray_objects which means no more objects have
1916          * been added. Note this is iterative: no recursion is involved.
1917          * We need to walk the LO list as well in search of marked big objects
1918          * (use a flag since this is needed only on major collections). We need to loop
1919          * here as well, so keep a counter of marked LO (increasing it in copy_object).
1920          *   To achieve better cache locality and cache usage, we drain the gray stack
1921          * frequently, after each object is copied, and just finish the work here.
1922          */
1923         sgen_drain_gray_stack (-1, ctx);
1924         TV_GETTIME (atv);
1925         SGEN_LOG (2, "%s generation done", generation_name (generation));
1926
1927         /*
1928         Reset bridge data, we might have lingering data from a previous collection if this is a major
1929         collection trigged by minor overflow.
1930
1931         We must reset the gathered bridges since their original block might be evacuated due to major
1932         fragmentation in the meanwhile and the bridge code should not have to deal with that.
1933         */
1934         sgen_bridge_reset_data ();
1935
1936         /*
1937          * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1938          * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1939          * objects that are in fact reachable.
1940          */
1941         done_with_ephemerons = 0;
1942         do {
1943                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1944                 sgen_drain_gray_stack (-1, ctx);
1945                 ++ephemeron_rounds;
1946         } while (!done_with_ephemerons);
1947
1948         sgen_scan_togglerefs (start_addr, end_addr, ctx);
1949         if (generation == GENERATION_OLD)
1950                 sgen_scan_togglerefs (sgen_get_nursery_start (), sgen_get_nursery_end (), ctx);
1951
1952         if (sgen_need_bridge_processing ()) {
1953                 sgen_collect_bridge_objects (generation, ctx);
1954                 if (generation == GENERATION_OLD)
1955                         sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1956         }
1957
1958         /*
1959         Make sure we drain the gray stack before processing disappearing links and finalizers.
1960         If we don't make sure it is empty we might wrongly see a live object as dead.
1961         */
1962         sgen_drain_gray_stack (-1, ctx);
1963
1964         /*
1965         We must clear weak links that don't track resurrection before processing object ready for
1966         finalization so they can be cleared before that.
1967         */
1968         sgen_null_link_in_range (generation, TRUE, ctx);
1969         if (generation == GENERATION_OLD)
1970                 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1971
1972
1973         /* walk the finalization queue and move also the objects that need to be
1974          * finalized: use the finalized objects as new roots so the objects they depend
1975          * on are also not reclaimed. As with the roots above, only objects in the nursery
1976          * are marked/copied.
1977          */
1978         sgen_finalize_in_range (generation, ctx);
1979         if (generation == GENERATION_OLD)
1980                 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1981         /* drain the new stack that might have been created */
1982         SGEN_LOG (6, "Precise scan of gray area post fin");
1983         sgen_drain_gray_stack (-1, ctx);
1984
1985         /*
1986          * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1987          */
1988         done_with_ephemerons = 0;
1989         do {
1990                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1991                 sgen_drain_gray_stack (-1, ctx);
1992                 ++ephemeron_rounds;
1993         } while (!done_with_ephemerons);
1994
1995         /*
1996          * Clear ephemeron pairs with unreachable keys.
1997          * We pass the copy func so we can figure out if an array was promoted or not.
1998          */
1999         clear_unreachable_ephemerons (generation == GENERATION_OLD && major_collector.is_concurrent, ctx);
2000
2001         TV_GETTIME (btv);
2002         SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
2003
2004         /*
2005          * handle disappearing links
2006          * Note we do this after checking the finalization queue because if an object
2007          * survives (at least long enough to be finalized) we don't clear the link.
2008          * This also deals with a possible issue with the monitor reclamation: with the Boehm
2009          * GC a finalized object my lose the monitor because it is cleared before the finalizer is
2010          * called.
2011          */
2012         g_assert (sgen_gray_object_queue_is_empty (queue));
2013         for (;;) {
2014                 sgen_null_link_in_range (generation, FALSE, ctx);
2015                 if (generation == GENERATION_OLD)
2016                         sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
2017                 if (sgen_gray_object_queue_is_empty (queue))
2018                         break;
2019                 sgen_drain_gray_stack (-1, ctx);
2020         }
2021
2022         g_assert (sgen_gray_object_queue_is_empty (queue));
2023 }
2024
2025 void
2026 sgen_check_section_scan_starts (GCMemSection *section)
2027 {
2028         int i;
2029         for (i = 0; i < section->num_scan_start; ++i) {
2030                 if (section->scan_starts [i]) {
2031                         guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2032                         g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2033                 }
2034         }
2035 }
2036
2037 static void
2038 check_scan_starts (void)
2039 {
2040         if (!do_scan_starts_check)
2041                 return;
2042         sgen_check_section_scan_starts (nursery_section);
2043         major_collector.check_scan_starts ();
2044 }
2045
2046 static void
2047 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
2048 {
2049         void **start_root;
2050         RootRecord *root;
2051         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
2052                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
2053                 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
2054         } SGEN_HASH_TABLE_FOREACH_END;
2055 }
2056
2057 void
2058 sgen_dump_occupied (char *start, char *end, char *section_start)
2059 {
2060         fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2061 }
2062
2063 void
2064 sgen_dump_section (GCMemSection *section, const char *type)
2065 {
2066         char *start = section->data;
2067         char *end = section->data + section->size;
2068         char *occ_start = NULL;
2069         GCVTable *vt;
2070         char *old_start = NULL; /* just for debugging */
2071
2072         fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2073
2074         while (start < end) {
2075                 guint size;
2076                 MonoClass *class;
2077
2078                 if (!*(void**)start) {
2079                         if (occ_start) {
2080                                 sgen_dump_occupied (occ_start, start, section->data);
2081                                 occ_start = NULL;
2082                         }
2083                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2084                         continue;
2085                 }
2086                 g_assert (start < section->next_data);
2087
2088                 if (!occ_start)
2089                         occ_start = start;
2090
2091                 vt = (GCVTable*)LOAD_VTABLE (start);
2092                 class = vt->klass;
2093
2094                 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2095
2096                 /*
2097                 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2098                                 start - section->data,
2099                                 vt->klass->name_space, vt->klass->name,
2100                                 size);
2101                 */
2102
2103                 old_start = start;
2104                 start += size;
2105         }
2106         if (occ_start)
2107                 sgen_dump_occupied (occ_start, start, section->data);
2108
2109         fprintf (heap_dump_file, "</section>\n");
2110 }
2111
2112 static void
2113 dump_object (MonoObject *obj, gboolean dump_location)
2114 {
2115         static char class_name [1024];
2116
2117         MonoClass *class = mono_object_class (obj);
2118         int i, j;
2119
2120         /*
2121          * Python's XML parser is too stupid to parse angle brackets
2122          * in strings, so we just ignore them;
2123          */
2124         i = j = 0;
2125         while (class->name [i] && j < sizeof (class_name) - 1) {
2126                 if (!strchr ("<>\"", class->name [i]))
2127                         class_name [j++] = class->name [i];
2128                 ++i;
2129         }
2130         g_assert (j < sizeof (class_name));
2131         class_name [j] = 0;
2132
2133         fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2134                         class->name_space, class_name,
2135                         safe_object_get_size (obj));
2136         if (dump_location) {
2137                 const char *location;
2138                 if (ptr_in_nursery (obj))
2139                         location = "nursery";
2140                 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2141                         location = "major";
2142                 else
2143                         location = "LOS";
2144                 fprintf (heap_dump_file, " location=\"%s\"", location);
2145         }
2146         fprintf (heap_dump_file, "/>\n");
2147 }
2148
2149 static void
2150 dump_heap (const char *type, int num, const char *reason)
2151 {
2152         ObjectList *list;
2153         LOSObject *bigobj;
2154
2155         fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2156         if (reason)
2157                 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2158         fprintf (heap_dump_file, ">\n");
2159         fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2160         sgen_dump_internal_mem_usage (heap_dump_file);
2161         fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
2162         /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2163         fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
2164
2165         fprintf (heap_dump_file, "<pinned-objects>\n");
2166         for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
2167                 dump_object (list->obj, TRUE);
2168         fprintf (heap_dump_file, "</pinned-objects>\n");
2169
2170         sgen_dump_section (nursery_section, "nursery");
2171
2172         major_collector.dump_heap (heap_dump_file);
2173
2174         fprintf (heap_dump_file, "<los>\n");
2175         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2176                 dump_object ((MonoObject*)bigobj->data, FALSE);
2177         fprintf (heap_dump_file, "</los>\n");
2178
2179         fprintf (heap_dump_file, "</collection>\n");
2180 }
2181
2182 void
2183 sgen_register_moved_object (void *obj, void *destination)
2184 {
2185         g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
2186
2187         /* FIXME: handle this for parallel collector */
2188         g_assert (!sgen_collection_is_parallel ());
2189
2190         if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2191                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2192                 moved_objects_idx = 0;
2193         }
2194         moved_objects [moved_objects_idx++] = obj;
2195         moved_objects [moved_objects_idx++] = destination;
2196 }
2197
2198 static void
2199 init_stats (void)
2200 {
2201         static gboolean inited = FALSE;
2202
2203         if (inited)
2204                 return;
2205
2206         mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pre_collection_fragment_clear);
2207         mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pinning);
2208         mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_remsets);
2209         mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_pinned);
2210         mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_registered_roots);
2211         mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_thread_data);
2212         mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_finish_gray_stack);
2213         mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_fragment_creation);
2214
2215         mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pre_collection_fragment_clear);
2216         mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pinning);
2217         mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_pinned);
2218         mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_registered_roots);
2219         mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_thread_data);
2220         mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_alloc_pinned);
2221         mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_finalized);
2222         mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_big_objects);
2223         mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_finish_gray_stack);
2224         mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_free_bigobjs);
2225         mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_los_sweep);
2226         mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_sweep);
2227         mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_fragment_creation);
2228
2229         mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_pinned_objects);
2230
2231 #ifdef HEAVY_STATISTICS
2232         mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
2233         mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
2234         mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
2235         mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
2236         mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
2237         mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
2238         mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
2239
2240         mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
2241         mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
2242
2243         mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
2244         mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
2245         mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
2246         mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
2247
2248         mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
2249         mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
2250
2251         mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
2252
2253         mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
2254         mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
2255         mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
2256         mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_to_space);
2257
2258         sgen_nursery_allocator_init_heavy_stats ();
2259         sgen_alloc_init_heavy_stats ();
2260 #endif
2261
2262         inited = TRUE;
2263 }
2264
2265
2266 static void
2267 reset_pinned_from_failed_allocation (void)
2268 {
2269         bytes_pinned_from_failed_allocation = 0;
2270 }
2271
2272 void
2273 sgen_set_pinned_from_failed_allocation (mword objsize)
2274 {
2275         bytes_pinned_from_failed_allocation += objsize;
2276 }
2277
2278 gboolean
2279 sgen_collection_is_parallel (void)
2280 {
2281         switch (current_collection_generation) {
2282         case GENERATION_NURSERY:
2283                 return nursery_collection_is_parallel;
2284         case GENERATION_OLD:
2285                 return major_collector.is_parallel;
2286         default:
2287                 g_error ("Invalid current generation %d", current_collection_generation);
2288         }
2289 }
2290
2291 gboolean
2292 sgen_collection_is_concurrent (void)
2293 {
2294         switch (current_collection_generation) {
2295         case GENERATION_NURSERY:
2296                 return FALSE;
2297         case GENERATION_OLD:
2298                 return major_collector.is_concurrent;
2299         default:
2300                 g_error ("Invalid current generation %d", current_collection_generation);
2301         }
2302 }
2303
2304 gboolean
2305 sgen_concurrent_collection_in_progress (void)
2306 {
2307         return concurrent_collection_in_progress;
2308 }
2309
2310 typedef struct
2311 {
2312         char *heap_start;
2313         char *heap_end;
2314 } FinishRememberedSetScanJobData;
2315
2316 static void
2317 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2318 {
2319         FinishRememberedSetScanJobData *job_data = job_data_untyped;
2320
2321         remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2322         sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2323 }
2324
2325 typedef struct
2326 {
2327         CopyOrMarkObjectFunc copy_or_mark_func;
2328         ScanObjectFunc scan_func;
2329         char *heap_start;
2330         char *heap_end;
2331         int root_type;
2332 } ScanFromRegisteredRootsJobData;
2333
2334 static void
2335 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2336 {
2337         ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2338         ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2339                 sgen_workers_get_job_gray_queue (worker_data) };
2340
2341         scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2342         sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2343 }
2344
2345 typedef struct
2346 {
2347         char *heap_start;
2348         char *heap_end;
2349 } ScanThreadDataJobData;
2350
2351 static void
2352 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2353 {
2354         ScanThreadDataJobData *job_data = job_data_untyped;
2355
2356         scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2357                         sgen_workers_get_job_gray_queue (worker_data));
2358         sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2359 }
2360
2361 typedef struct
2362 {
2363         FinalizeReadyEntry *list;
2364 } ScanFinalizerEntriesJobData;
2365
2366 static void
2367 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2368 {
2369         ScanFinalizerEntriesJobData *job_data = job_data_untyped;
2370         ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2371
2372         scan_finalizer_entries (job_data->list, ctx);
2373         sgen_free_internal_dynamic (job_data, sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2374 }
2375
2376 static void
2377 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2378 {
2379         g_assert (concurrent_collection_in_progress);
2380         major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2381 }
2382
2383 static void
2384 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2385 {
2386         g_assert (concurrent_collection_in_progress);
2387         sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2388 }
2389
2390 static void
2391 verify_scan_starts (char *start, char *end)
2392 {
2393         int i;
2394
2395         for (i = 0; i < nursery_section->num_scan_start; ++i) {
2396                 char *addr = nursery_section->scan_starts [i];
2397                 if (addr > start && addr < end)
2398                         SGEN_LOG (1, "NFC-BAD SCAN START [%d] %p for obj [%p %p]", i, addr, start, end);
2399         }
2400 }
2401
2402 static void
2403 verify_nursery (void)
2404 {
2405         char *start, *end, *cur, *hole_start;
2406
2407         if (!do_verify_nursery)
2408                 return;
2409
2410         /*This cleans up unused fragments */
2411         sgen_nursery_allocator_prepare_for_pinning ();
2412
2413         hole_start = start = cur = sgen_get_nursery_start ();
2414         end = sgen_get_nursery_end ();
2415
2416         while (cur < end) {
2417                 size_t ss, size;
2418
2419                 if (!*(void**)cur) {
2420                         cur += sizeof (void*);
2421                         continue;
2422                 }
2423
2424                 if (object_is_forwarded (cur))
2425                         SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2426                 else if (object_is_pinned (cur))
2427                         SGEN_LOG (1, "PINNED OBJ %p", cur);
2428
2429                 ss = safe_object_get_size ((MonoObject*)cur);
2430                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2431                 verify_scan_starts (cur, cur + size);
2432                 if (do_dump_nursery_content) {
2433                         if (cur > hole_start)
2434                                 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2435                         SGEN_LOG (1, "OBJ  [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2436                 }
2437                 cur += size;
2438                 hole_start = cur;
2439         }
2440 }
2441
2442 /*
2443  * Checks that no objects in the nursery are fowarded or pinned.  This
2444  * is a precondition to restarting the mutator while doing a
2445  * concurrent collection.  Note that we don't clear fragments because
2446  * we depend on that having happened earlier.
2447  */
2448 static void
2449 check_nursery_is_clean (void)
2450 {
2451         char *start, *end, *cur;
2452
2453         start = cur = sgen_get_nursery_start ();
2454         end = sgen_get_nursery_end ();
2455
2456         while (cur < end) {
2457                 size_t ss, size;
2458
2459                 if (!*(void**)cur) {
2460                         cur += sizeof (void*);
2461                         continue;
2462                 }
2463
2464                 g_assert (!object_is_forwarded (cur));
2465                 g_assert (!object_is_pinned (cur));
2466
2467                 ss = safe_object_get_size ((MonoObject*)cur);
2468                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2469                 verify_scan_starts (cur, cur + size);
2470
2471                 cur += size;
2472         }
2473 }
2474
2475 static void
2476 init_gray_queue (void)
2477 {
2478         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) {
2479                 sgen_workers_init_distribute_gray_queue ();
2480                 sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
2481                                 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2482         } else {
2483                 sgen_gray_object_queue_init (&gray_queue, NULL);
2484         }
2485
2486         if (major_collector.is_concurrent) {
2487                 sgen_gray_object_queue_init_with_alloc_prepare (&remember_major_objects_gray_queue, NULL,
2488                                 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2489         } else {
2490                 sgen_gray_object_queue_init_invalid (&remember_major_objects_gray_queue);
2491         }
2492 }
2493
2494 static void
2495 pin_stage_object_callback (char *obj, size_t size, void *data)
2496 {
2497         sgen_pin_stage_ptr (obj);
2498         /* FIXME: do pin stats if enabled */
2499 }
2500
2501 /*
2502  * Collect objects in the nursery.  Returns whether to trigger a major
2503  * collection.
2504  */
2505 static gboolean
2506 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2507 {
2508         gboolean needs_major;
2509         size_t max_garbage_amount;
2510         char *nursery_next;
2511         FinishRememberedSetScanJobData *frssjd;
2512         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2513         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2514         ScanThreadDataJobData *stdjd;
2515         mword fragment_total;
2516         ScanCopyContext ctx;
2517         TV_DECLARE (all_atv);
2518         TV_DECLARE (all_btv);
2519         TV_DECLARE (atv);
2520         TV_DECLARE (btv);
2521
2522         if (disable_minor_collections)
2523                 return TRUE;
2524
2525         MONO_GC_BEGIN (GENERATION_NURSERY);
2526         binary_protocol_collection_begin (stat_minor_gcs, GENERATION_NURSERY);
2527
2528         verify_nursery ();
2529
2530 #ifndef DISABLE_PERFCOUNTERS
2531         mono_perfcounters->gc_collections0++;
2532 #endif
2533
2534         current_collection_generation = GENERATION_NURSERY;
2535         if (sgen_collection_is_parallel ())
2536                 current_object_ops = sgen_minor_collector.parallel_ops;
2537         else
2538                 current_object_ops = sgen_minor_collector.serial_ops;
2539
2540         reset_pinned_from_failed_allocation ();
2541
2542         check_scan_starts ();
2543
2544         sgen_nursery_alloc_prepare_for_minor ();
2545
2546         degraded_mode = 0;
2547         objects_pinned = 0;
2548         nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2549         /* FIXME: optimize later to use the higher address where an object can be present */
2550         nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2551
2552         SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", stat_minor_gcs, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2553         max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2554         g_assert (nursery_section->size >= max_garbage_amount);
2555
2556         /* world must be stopped already */
2557         TV_GETTIME (all_atv);
2558         atv = all_atv;
2559
2560         TV_GETTIME (btv);
2561         time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2562
2563         if (xdomain_checks) {
2564                 sgen_clear_nursery_fragments ();
2565                 check_for_xdomain_refs ();
2566         }
2567
2568         nursery_section->next_data = nursery_next;
2569
2570         major_collector.start_nursery_collection ();
2571
2572         sgen_memgov_minor_collection_start ();
2573
2574         init_gray_queue ();
2575
2576         stat_minor_gcs++;
2577         gc_stats.minor_gc_count ++;
2578
2579         if (remset.prepare_for_minor_collection)
2580                 remset.prepare_for_minor_collection ();
2581
2582         MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2583
2584         sgen_process_fin_stage_entries ();
2585         sgen_process_dislink_stage_entries ();
2586
2587         MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2588
2589         /* pin from pinned handles */
2590         sgen_init_pinning ();
2591         mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2592         pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2593         /* pin cemented objects */
2594         sgen_cement_iterate (pin_stage_object_callback, NULL);
2595         /* identify pinned objects */
2596         sgen_optimize_pin_queue (0);
2597         sgen_pinning_setup_section (nursery_section);
2598         ctx.scan_func = NULL;
2599         ctx.copy_func = NULL;
2600         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2601         sgen_pin_objects_in_section (nursery_section, ctx);
2602         sgen_pinning_trim_queue_to_section (nursery_section);
2603
2604         TV_GETTIME (atv);
2605         time_minor_pinning += TV_ELAPSED (btv, atv);
2606         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2607         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2608
2609         MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2610
2611         if (whole_heap_check_before_collection) {
2612                 sgen_clear_nursery_fragments ();
2613                 sgen_check_whole_heap (finish_up_concurrent_mark);
2614         }
2615         if (consistency_check_at_minor_collection)
2616                 sgen_check_consistency ();
2617
2618         sgen_workers_start_all_workers ();
2619
2620         /*
2621          * Perform the sequential part of remembered set scanning.
2622          * This usually involves scanning global information that might later be produced by evacuation.
2623          */
2624         if (remset.begin_scan_remsets)
2625                 remset.begin_scan_remsets (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2626
2627         sgen_workers_start_marking ();
2628
2629         frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2630         frssjd->heap_start = sgen_get_nursery_start ();
2631         frssjd->heap_end = nursery_next;
2632         sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2633
2634         /* we don't have complete write barrier yet, so we scan all the old generation sections */
2635         TV_GETTIME (btv);
2636         time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2637         SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2638
2639         MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2640
2641         if (!sgen_collection_is_parallel ()) {
2642                 ctx.scan_func = current_object_ops.scan_object;
2643                 ctx.copy_func = NULL;
2644                 ctx.queue = &gray_queue;
2645                 sgen_drain_gray_stack (-1, ctx);
2646         }
2647
2648         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2649                 report_registered_roots ();
2650         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2651                 report_finalizer_roots ();
2652         TV_GETTIME (atv);
2653         time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2654
2655         MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2656
2657         /* registered roots, this includes static fields */
2658         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2659         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2660         scrrjd_normal->scan_func = current_object_ops.scan_object;
2661         scrrjd_normal->heap_start = sgen_get_nursery_start ();
2662         scrrjd_normal->heap_end = nursery_next;
2663         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2664         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2665
2666         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2667         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2668         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2669         scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2670         scrrjd_wbarrier->heap_end = nursery_next;
2671         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2672         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2673
2674         TV_GETTIME (btv);
2675         time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2676
2677         MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2678
2679         /* thread data */
2680         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2681         stdjd->heap_start = sgen_get_nursery_start ();
2682         stdjd->heap_end = nursery_next;
2683         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2684
2685         TV_GETTIME (atv);
2686         time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2687         btv = atv;
2688
2689         MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2690
2691         g_assert (!sgen_collection_is_parallel () && !sgen_collection_is_concurrent ());
2692
2693         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ())
2694                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2695
2696         /* Scan the list of objects ready for finalization. If */
2697         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2698         sfejd_fin_ready->list = fin_ready_list;
2699         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
2700
2701         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2702         sfejd_critical_fin->list = critical_fin_list;
2703         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
2704
2705         MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2706
2707         finish_gray_stack (sgen_get_nursery_start (), nursery_next, GENERATION_NURSERY, &gray_queue);
2708         TV_GETTIME (atv);
2709         time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2710         mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2711
2712         MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2713
2714         /*
2715          * The (single-threaded) finalization code might have done
2716          * some copying/marking so we can only reset the GC thread's
2717          * worker data here instead of earlier when we joined the
2718          * workers.
2719          */
2720         sgen_workers_reset_data ();
2721
2722         if (objects_pinned) {
2723                 sgen_optimize_pin_queue (0);
2724                 sgen_pinning_setup_section (nursery_section);
2725         }
2726
2727         /* walk the pin_queue, build up the fragment list of free memory, unmark
2728          * pinned objects as we go, memzero() the empty fragments so they are ready for the
2729          * next allocations.
2730          */
2731         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2732         fragment_total = sgen_build_nursery_fragments (nursery_section,
2733                         nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries,
2734                         unpin_queue);
2735         if (!fragment_total)
2736                 degraded_mode = 1;
2737
2738         /* Clear TLABs for all threads */
2739         sgen_clear_tlabs ();
2740
2741         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2742         TV_GETTIME (btv);
2743         time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2744         SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2745
2746         if (consistency_check_at_minor_collection)
2747                 sgen_check_major_refs ();
2748
2749         major_collector.finish_nursery_collection ();
2750
2751         TV_GETTIME (all_btv);
2752         gc_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2753
2754         if (heap_dump_file)
2755                 dump_heap ("minor", stat_minor_gcs - 1, NULL);
2756
2757         /* prepare the pin queue for the next collection */
2758         sgen_finish_pinning ();
2759         if (fin_ready_list || critical_fin_list) {
2760                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2761                 mono_gc_finalize_notify ();
2762         }
2763         sgen_pin_stats_reset ();
2764         /* clear cemented hash */
2765         sgen_cement_clear_below_threshold ();
2766
2767         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2768
2769         if (remset.finish_minor_collection)
2770                 remset.finish_minor_collection ();
2771
2772         check_scan_starts ();
2773
2774         binary_protocol_flush_buffers (FALSE);
2775
2776         sgen_memgov_minor_collection_end ();
2777
2778         /*objects are late pinned because of lack of memory, so a major is a good call*/
2779         needs_major = objects_pinned > 0;
2780         current_collection_generation = -1;
2781         objects_pinned = 0;
2782
2783         MONO_GC_END (GENERATION_NURSERY);
2784         binary_protocol_collection_end (stat_minor_gcs - 1, GENERATION_NURSERY);
2785
2786         if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2787                 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2788
2789         return needs_major;
2790 }
2791
2792 static void
2793 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2794 {
2795         ctx->scan_func (obj, ctx->queue);
2796 }
2797
2798 static void
2799 scan_nursery_objects (ScanCopyContext ctx)
2800 {
2801         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2802                         (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2803 }
2804
2805 static void
2806 major_copy_or_mark_from_roots (int *old_next_pin_slot, gboolean finish_up_concurrent_mark, gboolean scan_mod_union)
2807 {
2808         LOSObject *bigobj;
2809         TV_DECLARE (atv);
2810         TV_DECLARE (btv);
2811         /* FIXME: only use these values for the precise scan
2812          * note that to_space pointers should be excluded anyway...
2813          */
2814         char *heap_start = NULL;
2815         char *heap_end = (char*)-1;
2816         gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2817         GCRootReport root_report = { 0 };
2818         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2819         ScanThreadDataJobData *stdjd;
2820         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2821         ScanCopyContext ctx;
2822
2823         if (major_collector.is_concurrent) {
2824                 /*This cleans up unused fragments */
2825                 sgen_nursery_allocator_prepare_for_pinning ();
2826
2827                 if (do_concurrent_checks)
2828                         check_nursery_is_clean ();
2829         } else {
2830                 /* The concurrent collector doesn't touch the nursery. */
2831                 sgen_nursery_alloc_prepare_for_major ();
2832         }
2833
2834         init_gray_queue ();
2835
2836         TV_GETTIME (atv);
2837
2838         /* Pinning depends on this */
2839         sgen_clear_nursery_fragments ();
2840
2841         if (whole_heap_check_before_collection)
2842                 sgen_check_whole_heap (finish_up_concurrent_mark);
2843
2844         if (!major_collector.is_concurrent)
2845                 sgen_cement_reset ();
2846
2847         TV_GETTIME (btv);
2848         time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2849
2850         if (!sgen_collection_is_concurrent ())
2851                 nursery_section->next_data = sgen_get_nursery_end ();
2852         /* we should also coalesce scanning from sections close to each other
2853          * and deal with pointers outside of the sections later.
2854          */
2855
2856         objects_pinned = 0;
2857         *major_collector.have_swept = FALSE;
2858
2859         if (xdomain_checks) {
2860                 sgen_clear_nursery_fragments ();
2861                 check_for_xdomain_refs ();
2862         }
2863
2864         if (!major_collector.is_concurrent) {
2865                 /* Remsets are not useful for a major collection */
2866                 remset.prepare_for_major_collection ();
2867         }
2868
2869         sgen_process_fin_stage_entries ();
2870         sgen_process_dislink_stage_entries ();
2871
2872         TV_GETTIME (atv);
2873         sgen_init_pinning ();
2874         SGEN_LOG (6, "Collecting pinned addresses");
2875         pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2876         sgen_optimize_pin_queue (0);
2877
2878         /*
2879          * The concurrent collector doesn't move objects, neither on
2880          * the major heap nor in the nursery, so we can mark even
2881          * before pinning has finished.  For the non-concurrent
2882          * collector we start the workers after pinning.
2883          */
2884         if (major_collector.is_concurrent) {
2885                 sgen_workers_start_all_workers ();
2886                 sgen_workers_start_marking ();
2887         }
2888
2889         /*
2890          * pin_queue now contains all candidate pointers, sorted and
2891          * uniqued.  We must do two passes now to figure out which
2892          * objects are pinned.
2893          *
2894          * The first is to find within the pin_queue the area for each
2895          * section.  This requires that the pin_queue be sorted.  We
2896          * also process the LOS objects and pinned chunks here.
2897          *
2898          * The second, destructive, pass is to reduce the section
2899          * areas to pointers to the actually pinned objects.
2900          */
2901         SGEN_LOG (6, "Pinning from sections");
2902         /* first pass for the sections */
2903         sgen_find_section_pin_queue_start_end (nursery_section);
2904         major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2905         /* identify possible pointers to the insize of large objects */
2906         SGEN_LOG (6, "Pinning from large objects");
2907         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2908                 int dummy;
2909                 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) {
2910                         binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2911
2912 #ifdef ENABLE_DTRACE
2913                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2914                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2915                                 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2916                         }
2917 #endif
2918
2919                         if (sgen_los_object_is_pinned (bigobj->data)) {
2920                                 g_assert (finish_up_concurrent_mark);
2921                                 continue;
2922                         }
2923                         sgen_los_pin_object (bigobj->data);
2924                         /* FIXME: only enqueue if object has references */
2925                         GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data);
2926                         if (G_UNLIKELY (do_pin_stats))
2927                                 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2928                         SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2929
2930                         if (profile_roots)
2931                                 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2932                 }
2933         }
2934         if (profile_roots)
2935                 notify_gc_roots (&root_report);
2936         /* second pass for the sections */
2937         ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2938         ctx.copy_func = NULL;
2939         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2940
2941         /*
2942          * Concurrent mark never follows references into the nursery.
2943          * In the start and finish pauses we must scan live nursery
2944          * objects, though.  We could simply scan all nursery objects,
2945          * but that would be conservative.  The easiest way is to do a
2946          * nursery collection, which copies all live nursery objects
2947          * (except pinned ones, with the simple nursery) to the major
2948          * heap.  Scanning the mod union table later will then scan
2949          * those promoted objects, provided they're reachable.  Pinned
2950          * objects in the nursery - which we can trivially find in the
2951          * pinning queue - are treated as roots in the mark pauses.
2952          *
2953          * The split nursery complicates the latter part because
2954          * non-pinned objects can survive in the nursery.  That's why
2955          * we need to do a full front-to-back scan of the nursery,
2956          * marking all objects.
2957          *
2958          * Non-concurrent mark evacuates from the nursery, so it's
2959          * sufficient to just scan pinned nursery objects.
2960          */
2961         if (major_collector.is_concurrent && sgen_minor_collector.is_split) {
2962                 scan_nursery_objects (ctx);
2963         } else {
2964                 sgen_pin_objects_in_section (nursery_section, ctx);
2965                 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2966                         sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2967         }
2968
2969         major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2970         if (old_next_pin_slot)
2971                 *old_next_pin_slot = sgen_get_pinned_count ();
2972
2973         TV_GETTIME (btv);
2974         time_major_pinning += TV_ELAPSED (atv, btv);
2975         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2976         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2977
2978         major_collector.init_to_space ();
2979
2980 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2981         main_gc_thread = mono_native_thread_self ();
2982 #endif
2983
2984         if (!major_collector.is_concurrent) {
2985                 sgen_workers_start_all_workers ();
2986                 sgen_workers_start_marking ();
2987         }
2988
2989         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2990                 report_registered_roots ();
2991         TV_GETTIME (atv);
2992         time_major_scan_pinned += TV_ELAPSED (btv, atv);
2993
2994         /* registered roots, this includes static fields */
2995         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2996         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2997         scrrjd_normal->scan_func = current_object_ops.scan_object;
2998         scrrjd_normal->heap_start = heap_start;
2999         scrrjd_normal->heap_end = heap_end;
3000         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
3001         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
3002
3003         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3004         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
3005         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
3006         scrrjd_wbarrier->heap_start = heap_start;
3007         scrrjd_wbarrier->heap_end = heap_end;
3008         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
3009         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
3010
3011         TV_GETTIME (btv);
3012         time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
3013
3014         /* Threads */
3015         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3016         stdjd->heap_start = heap_start;
3017         stdjd->heap_end = heap_end;
3018         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
3019
3020         TV_GETTIME (atv);
3021         time_major_scan_thread_data += TV_ELAPSED (btv, atv);
3022
3023         TV_GETTIME (btv);
3024         time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
3025
3026         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
3027                 report_finalizer_roots ();
3028
3029         /* scan the list of objects ready for finalization */
3030         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3031         sfejd_fin_ready->list = fin_ready_list;
3032         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
3033
3034         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3035         sfejd_critical_fin->list = critical_fin_list;
3036         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
3037
3038         if (scan_mod_union) {
3039                 g_assert (finish_up_concurrent_mark);
3040
3041                 /* Mod union card table */
3042                 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
3043                 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
3044         }
3045
3046         TV_GETTIME (atv);
3047         time_major_scan_finalized += TV_ELAPSED (btv, atv);
3048         SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
3049
3050         TV_GETTIME (btv);
3051         time_major_scan_big_objects += TV_ELAPSED (atv, btv);
3052
3053         if (major_collector.is_concurrent) {
3054                 /* prepare the pin queue for the next collection */
3055                 sgen_finish_pinning ();
3056
3057                 sgen_pin_stats_reset ();
3058
3059                 if (do_concurrent_checks)
3060                         check_nursery_is_clean ();
3061         }
3062 }
3063
3064 static void
3065 major_start_collection (int *old_next_pin_slot)
3066 {
3067         MONO_GC_BEGIN (GENERATION_OLD);
3068         binary_protocol_collection_begin (stat_major_gcs, GENERATION_OLD);
3069
3070         current_collection_generation = GENERATION_OLD;
3071 #ifndef DISABLE_PERFCOUNTERS
3072         mono_perfcounters->gc_collections1++;
3073 #endif
3074
3075         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3076
3077         if (major_collector.is_concurrent) {
3078                 concurrent_collection_in_progress = TRUE;
3079
3080                 sgen_cement_concurrent_start ();
3081         }
3082
3083         current_object_ops = major_collector.major_ops;
3084
3085         reset_pinned_from_failed_allocation ();
3086
3087         sgen_memgov_major_collection_start ();
3088
3089         //count_ref_nonref_objs ();
3090         //consistency_check ();
3091
3092         check_scan_starts ();
3093
3094         degraded_mode = 0;
3095         SGEN_LOG (1, "Start major collection %d", stat_major_gcs);
3096         stat_major_gcs++;
3097         gc_stats.major_gc_count ++;
3098
3099         if (major_collector.start_major_collection)
3100                 major_collector.start_major_collection ();
3101
3102         major_copy_or_mark_from_roots (old_next_pin_slot, FALSE, FALSE);
3103 }
3104
3105 static void
3106 wait_for_workers_to_finish (void)
3107 {
3108         g_assert (sgen_gray_object_queue_is_empty (&remember_major_objects_gray_queue));
3109
3110         if (major_collector.is_parallel || major_collector.is_concurrent) {
3111                 gray_queue_redirect (&gray_queue);
3112                 sgen_workers_join ();
3113         }
3114
3115         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3116
3117 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
3118         main_gc_thread = NULL;
3119 #endif
3120 }
3121
3122 static void
3123 major_finish_collection (const char *reason, int old_next_pin_slot, gboolean scan_mod_union)
3124 {
3125         LOSObject *bigobj, *prevbo;
3126         TV_DECLARE (atv);
3127         TV_DECLARE (btv);
3128         char *heap_start = NULL;
3129         char *heap_end = (char*)-1;
3130
3131         TV_GETTIME (btv);
3132
3133         if (major_collector.is_concurrent || major_collector.is_parallel)
3134                 wait_for_workers_to_finish ();
3135
3136         current_object_ops = major_collector.major_ops;
3137
3138         if (major_collector.is_concurrent) {
3139                 major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
3140                 wait_for_workers_to_finish ();
3141
3142                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3143
3144                 if (do_concurrent_checks)
3145                         check_nursery_is_clean ();
3146         }
3147
3148         /*
3149          * The workers have stopped so we need to finish gray queue
3150          * work that might result from finalization in the main GC
3151          * thread.  Redirection must therefore be turned off.
3152          */
3153         sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
3154         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3155
3156         /* all the objects in the heap */
3157         finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
3158         TV_GETTIME (atv);
3159         time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
3160
3161         /*
3162          * The (single-threaded) finalization code might have done
3163          * some copying/marking so we can only reset the GC thread's
3164          * worker data here instead of earlier when we joined the
3165          * workers.
3166          */
3167         sgen_workers_reset_data ();
3168
3169         if (objects_pinned) {
3170                 g_assert (!major_collector.is_concurrent);
3171
3172                 /*This is slow, but we just OOM'd*/
3173                 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
3174                 sgen_optimize_pin_queue (0);
3175                 sgen_find_section_pin_queue_start_end (nursery_section);
3176                 objects_pinned = 0;
3177         }
3178
3179         reset_heap_boundaries ();
3180         sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
3181
3182         if (check_mark_bits_after_major_collection)
3183                 sgen_check_major_heap_marked ();
3184
3185         MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
3186
3187         /* sweep the big objects list */
3188         prevbo = NULL;
3189         for (bigobj = los_object_list; bigobj;) {
3190                 g_assert (!object_is_pinned (bigobj->data));
3191                 if (sgen_los_object_is_pinned (bigobj->data)) {
3192                         sgen_los_unpin_object (bigobj->data);
3193                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
3194                 } else {
3195                         LOSObject *to_free;
3196                         /* not referenced anywhere, so we can free it */
3197                         if (prevbo)
3198                                 prevbo->next = bigobj->next;
3199                         else
3200                                 los_object_list = bigobj->next;
3201                         to_free = bigobj;
3202                         bigobj = bigobj->next;
3203                         sgen_los_free_object (to_free);
3204                         continue;
3205                 }
3206                 prevbo = bigobj;
3207                 bigobj = bigobj->next;
3208         }
3209
3210         TV_GETTIME (btv);
3211         time_major_free_bigobjs += TV_ELAPSED (atv, btv);
3212
3213         sgen_los_sweep ();
3214
3215         TV_GETTIME (atv);
3216         time_major_los_sweep += TV_ELAPSED (btv, atv);
3217
3218         major_collector.sweep ();
3219
3220         MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
3221
3222         TV_GETTIME (btv);
3223         time_major_sweep += TV_ELAPSED (atv, btv);
3224
3225         if (!major_collector.is_concurrent) {
3226                 /* walk the pin_queue, build up the fragment list of free memory, unmark
3227                  * pinned objects as we go, memzero() the empty fragments so they are ready for the
3228                  * next allocations.
3229                  */
3230                 if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries, NULL))
3231                         degraded_mode = 1;
3232
3233                 /* prepare the pin queue for the next collection */
3234                 sgen_finish_pinning ();
3235
3236                 /* Clear TLABs for all threads */
3237                 sgen_clear_tlabs ();
3238
3239                 sgen_pin_stats_reset ();
3240         }
3241
3242         if (major_collector.is_concurrent)
3243                 sgen_cement_concurrent_finish ();
3244         sgen_cement_clear_below_threshold ();
3245
3246         TV_GETTIME (atv);
3247         time_major_fragment_creation += TV_ELAPSED (btv, atv);
3248
3249         if (heap_dump_file)
3250                 dump_heap ("major", stat_major_gcs - 1, reason);
3251
3252         if (fin_ready_list || critical_fin_list) {
3253                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
3254                 mono_gc_finalize_notify ();
3255         }
3256
3257         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3258
3259         sgen_memgov_major_collection_end ();
3260         current_collection_generation = -1;
3261
3262         major_collector.finish_major_collection ();
3263
3264         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3265
3266         if (major_collector.is_concurrent)
3267                 concurrent_collection_in_progress = FALSE;
3268
3269         check_scan_starts ();
3270
3271         binary_protocol_flush_buffers (FALSE);
3272
3273         //consistency_check ();
3274
3275         MONO_GC_END (GENERATION_OLD);
3276         binary_protocol_collection_end (stat_major_gcs - 1, GENERATION_OLD);
3277 }
3278
3279 static gboolean
3280 major_do_collection (const char *reason)
3281 {
3282         TV_DECLARE (all_atv);
3283         TV_DECLARE (all_btv);
3284         int old_next_pin_slot;
3285
3286         if (major_collector.get_and_reset_num_major_objects_marked) {
3287                 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
3288                 g_assert (!num_marked);
3289         }
3290
3291         /* world must be stopped already */
3292         TV_GETTIME (all_atv);
3293
3294         major_start_collection (&old_next_pin_slot);
3295         major_finish_collection (reason, old_next_pin_slot, FALSE);
3296
3297         TV_GETTIME (all_btv);
3298         gc_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3299
3300         /* FIXME: also report this to the user, preferably in gc-end. */
3301         if (major_collector.get_and_reset_num_major_objects_marked)
3302                 major_collector.get_and_reset_num_major_objects_marked ();
3303
3304         return bytes_pinned_from_failed_allocation > 0;
3305 }
3306
3307 static gboolean major_do_collection (const char *reason);
3308
3309 static void
3310 major_start_concurrent_collection (const char *reason)
3311 {
3312         long long num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3313
3314         g_assert (num_objects_marked == 0);
3315
3316         MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
3317
3318         // FIXME: store reason and pass it when finishing
3319         major_start_collection (NULL);
3320
3321         gray_queue_redirect (&gray_queue);
3322         sgen_workers_wait_for_jobs ();
3323
3324         num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3325         MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
3326
3327         current_collection_generation = -1;
3328 }
3329
3330 static gboolean
3331 major_update_or_finish_concurrent_collection (gboolean force_finish)
3332 {
3333         SgenGrayQueue unpin_queue;
3334         memset (&unpin_queue, 0, sizeof (unpin_queue));
3335
3336         MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3337
3338         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3339         if (!have_non_collection_major_object_remembers)
3340                 g_assert (sgen_gray_object_queue_is_empty (&remember_major_objects_gray_queue));
3341
3342         major_collector.update_cardtable_mod_union ();
3343         sgen_los_update_cardtable_mod_union ();
3344
3345         if (!force_finish && !sgen_workers_all_done ()) {
3346                 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3347                 return FALSE;
3348         }
3349
3350         collect_nursery (&unpin_queue, TRUE);
3351         redirect_major_object_remembers ();
3352
3353         current_collection_generation = GENERATION_OLD;
3354         major_finish_collection ("finishing", -1, TRUE);
3355
3356         if (whole_heap_check_before_collection)
3357                 sgen_check_whole_heap (FALSE);
3358
3359         unpin_objects_from_queue (&unpin_queue);
3360         sgen_gray_object_queue_deinit (&unpin_queue);
3361
3362         MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3363
3364         current_collection_generation = -1;
3365
3366         return TRUE;
3367 }
3368
3369 /*
3370  * Ensure an allocation request for @size will succeed by freeing enough memory.
3371  *
3372  * LOCKING: The GC lock MUST be held.
3373  */
3374 void
3375 sgen_ensure_free_space (size_t size)
3376 {
3377         int generation_to_collect = -1;
3378         const char *reason = NULL;
3379
3380
3381         if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3382                 if (sgen_need_major_collection (size)) {
3383                         reason = "LOS overflow";
3384                         generation_to_collect = GENERATION_OLD;
3385                 }
3386         } else {
3387                 if (degraded_mode) {
3388                         if (sgen_need_major_collection (size)) {
3389                                 reason = "Degraded mode overflow";
3390                                 generation_to_collect = GENERATION_OLD;
3391                         }
3392                 } else if (sgen_need_major_collection (size)) {
3393                         reason = "Minor allowance";
3394                         generation_to_collect = GENERATION_OLD;
3395                 } else {
3396                         generation_to_collect = GENERATION_NURSERY;
3397                         reason = "Nursery full";
3398                 }
3399         }
3400
3401         if (generation_to_collect == -1) {
3402                 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3403                         generation_to_collect = GENERATION_OLD;
3404                         reason = "Finish concurrent collection";
3405                 }
3406         }
3407
3408         if (generation_to_collect == -1)
3409                 return;
3410         sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3411 }
3412
3413 void
3414 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3415 {
3416         TV_DECLARE (gc_end);
3417         GGTimingInfo infos [2];
3418         int overflow_generation_to_collect = -1;
3419         int oldest_generation_collected = generation_to_collect;
3420         const char *overflow_reason = NULL;
3421
3422         MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3423
3424         g_assert (generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD);
3425
3426         if (have_non_collection_major_object_remembers) {
3427                 g_assert (concurrent_collection_in_progress);
3428                 redirect_major_object_remembers ();
3429         }
3430
3431         memset (infos, 0, sizeof (infos));
3432         mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3433
3434         infos [0].generation = generation_to_collect;
3435         infos [0].reason = reason;
3436         infos [0].is_overflow = FALSE;
3437         TV_GETTIME (infos [0].total_time);
3438         infos [1].generation = -1;
3439
3440         sgen_stop_world (generation_to_collect);
3441
3442         if (concurrent_collection_in_progress) {
3443                 if (major_update_or_finish_concurrent_collection (wait_to_finish && generation_to_collect == GENERATION_OLD)) {
3444                         oldest_generation_collected = GENERATION_OLD;
3445                         goto done;
3446                 }
3447                 if (generation_to_collect == GENERATION_OLD)
3448                         goto done;
3449         }
3450
3451         //FIXME extract overflow reason
3452         if (generation_to_collect == GENERATION_NURSERY) {
3453                 if (collect_nursery (NULL, FALSE)) {
3454                         overflow_generation_to_collect = GENERATION_OLD;
3455                         overflow_reason = "Minor overflow";
3456                 }
3457                 if (concurrent_collection_in_progress) {
3458                         redirect_major_object_remembers ();
3459                         sgen_workers_wake_up_all ();
3460                 }
3461         } else {
3462                 SgenGrayQueue unpin_queue;
3463                 SgenGrayQueue *unpin_queue_ptr;
3464                 memset (&unpin_queue, 0, sizeof (unpin_queue));
3465
3466                 if (major_collector.is_concurrent && wait_to_finish)
3467                         unpin_queue_ptr = &unpin_queue;
3468                 else
3469                         unpin_queue_ptr = NULL;
3470
3471                 if (major_collector.is_concurrent) {
3472                         g_assert (!concurrent_collection_in_progress);
3473                         collect_nursery (unpin_queue_ptr, FALSE);
3474                 }
3475
3476                 if (major_collector.is_concurrent && !wait_to_finish) {
3477                         major_start_concurrent_collection (reason);
3478                         // FIXME: set infos[0] properly
3479                         goto done;
3480                 } else {
3481                         if (major_do_collection (reason)) {
3482                                 overflow_generation_to_collect = GENERATION_NURSERY;
3483                                 overflow_reason = "Excessive pinning";
3484                         }
3485                 }
3486
3487                 if (unpin_queue_ptr) {
3488                         unpin_objects_from_queue (unpin_queue_ptr);
3489                         sgen_gray_object_queue_deinit (unpin_queue_ptr);
3490                 }
3491         }
3492
3493         TV_GETTIME (gc_end);
3494         infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end);
3495
3496
3497         if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) {
3498                 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3499                 infos [1].generation = overflow_generation_to_collect;
3500                 infos [1].reason = overflow_reason;
3501                 infos [1].is_overflow = TRUE;
3502                 infos [1].total_time = gc_end;
3503
3504                 if (overflow_generation_to_collect == GENERATION_NURSERY)
3505                         collect_nursery (NULL, FALSE);
3506                 else
3507                         major_do_collection (overflow_reason);
3508
3509                 TV_GETTIME (gc_end);
3510                 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3511
3512                 /* keep events symmetric */
3513                 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3514
3515                 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3516         }
3517
3518         SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3519
3520         /* this also sets the proper pointers for the next allocation */
3521         if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3522                 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3523                 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%d pinned)", requested_size, sgen_get_pinned_count ());
3524                 sgen_dump_pin_queue ();
3525                 degraded_mode = 1;
3526         }
3527
3528  done:
3529         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3530         g_assert (sgen_gray_object_queue_is_empty (&remember_major_objects_gray_queue));
3531
3532         sgen_restart_world (oldest_generation_collected, infos);
3533
3534         mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3535 }
3536
3537 /*
3538  * ######################################################################
3539  * ########  Memory allocation from the OS
3540  * ######################################################################
3541  * This section of code deals with getting memory from the OS and
3542  * allocating memory for GC-internal data structures.
3543  * Internal memory can be handled with a freelist for small objects.
3544  */
3545
3546 /*
3547  * Debug reporting.
3548  */
3549 G_GNUC_UNUSED static void
3550 report_internal_mem_usage (void)
3551 {
3552         printf ("Internal memory usage:\n");
3553         sgen_report_internal_mem_usage ();
3554         printf ("Pinned memory usage:\n");
3555         major_collector.report_pinned_memory_usage ();
3556 }
3557
3558 /*
3559  * ######################################################################
3560  * ########  Finalization support
3561  * ######################################################################
3562  */
3563
3564 static inline gboolean
3565 sgen_major_is_object_alive (void *object)
3566 {
3567         mword objsize;
3568
3569         /* Oldgen objects can be pinned and forwarded too */
3570         if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3571                 return TRUE;
3572
3573         /*
3574          * FIXME: major_collector.is_object_live() also calculates the
3575          * size.  Avoid the double calculation.
3576          */
3577         objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3578         if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3579                 return sgen_los_object_is_pinned (object);
3580
3581         return major_collector.is_object_live (object);
3582 }
3583
3584 /*
3585  * If the object has been forwarded it means it's still referenced from a root.
3586  * If it is pinned it's still alive as well.
3587  * A LOS object is only alive if we have pinned it.
3588  * Return TRUE if @obj is ready to be finalized.
3589  */
3590 static inline gboolean
3591 sgen_is_object_alive (void *object)
3592 {
3593         if (ptr_in_nursery (object))
3594                 return sgen_nursery_is_object_alive (object);
3595
3596         return sgen_major_is_object_alive (object);
3597 }
3598
3599 /*
3600  * This function returns true if @object is either alive or it belongs to the old gen
3601  * and we're currently doing a minor collection.
3602  */
3603 static inline int
3604 sgen_is_object_alive_for_current_gen (char *object)
3605 {
3606         if (ptr_in_nursery (object))
3607                 return sgen_nursery_is_object_alive (object);
3608
3609         if (current_collection_generation == GENERATION_NURSERY)
3610                 return TRUE;
3611
3612         return sgen_major_is_object_alive (object);
3613 }
3614
3615 /*
3616  * This function returns true if @object is either alive and belongs to the
3617  * current collection - major collections are full heap, so old gen objects
3618  * are never alive during a minor collection.
3619  */
3620 static inline int
3621 sgen_is_object_alive_and_on_current_collection (char *object)
3622 {
3623         if (ptr_in_nursery (object))
3624                 return sgen_nursery_is_object_alive (object);
3625
3626         if (current_collection_generation == GENERATION_NURSERY)
3627                 return FALSE;
3628
3629         return sgen_major_is_object_alive (object);
3630 }
3631
3632
3633 gboolean
3634 sgen_gc_is_object_ready_for_finalization (void *object)
3635 {
3636         return !sgen_is_object_alive (object);
3637 }
3638
3639 static gboolean
3640 has_critical_finalizer (MonoObject *obj)
3641 {
3642         MonoClass *class;
3643
3644         if (!mono_defaults.critical_finalizer_object)
3645                 return FALSE;
3646
3647         class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3648
3649         return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3650 }
3651
3652 void
3653 sgen_queue_finalization_entry (MonoObject *obj)
3654 {
3655         FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3656         gboolean critical = has_critical_finalizer (obj);
3657         entry->object = obj;
3658         if (critical) {
3659                 entry->next = critical_fin_list;
3660                 critical_fin_list = entry;
3661         } else {
3662                 entry->next = fin_ready_list;
3663                 fin_ready_list = entry;
3664         }
3665
3666 #ifdef ENABLE_DTRACE
3667         if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3668                 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3669                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3670                 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3671                                 vt->klass->name_space, vt->klass->name, gen, critical);
3672         }
3673 #endif
3674 }
3675
3676 gboolean
3677 sgen_object_is_live (void *obj)
3678 {
3679         return sgen_is_object_alive_and_on_current_collection (obj);
3680 }
3681
3682 /* LOCKING: requires that the GC lock is held */
3683 static void
3684 null_ephemerons_for_domain (MonoDomain *domain)
3685 {
3686         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3687
3688         while (current) {
3689                 MonoObject *object = (MonoObject*)current->array;
3690
3691                 if (object && !object->vtable) {
3692                         EphemeronLinkNode *tmp = current;
3693
3694                         if (prev)
3695                                 prev->next = current->next;
3696                         else
3697                                 ephemeron_list = current->next;
3698
3699                         current = current->next;
3700                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3701                 } else {
3702                         prev = current;
3703                         current = current->next;
3704                 }
3705         }
3706 }
3707
3708 /* LOCKING: requires that the GC lock is held */
3709 static void
3710 clear_unreachable_ephemerons (gboolean concurrent_cementing, ScanCopyContext ctx)
3711 {
3712         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3713         GrayQueue *queue = ctx.queue;
3714         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3715         MonoArray *array;
3716         Ephemeron *cur, *array_end;
3717         char *tombstone;
3718
3719         while (current) {
3720                 char *object = current->array;
3721
3722                 if (!sgen_is_object_alive_for_current_gen (object)) {
3723                         EphemeronLinkNode *tmp = current;
3724
3725                         SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3726
3727                         if (prev)
3728                                 prev->next = current->next;
3729                         else
3730                                 ephemeron_list = current->next;
3731
3732                         current = current->next;
3733                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3734
3735                         continue;
3736                 }
3737
3738                 copy_func ((void**)&object, queue);
3739                 current->array = object;
3740
3741                 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3742
3743                 array = (MonoArray*)object;
3744                 cur = mono_array_addr (array, Ephemeron, 0);
3745                 array_end = cur + mono_array_length_fast (array);
3746                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3747
3748                 for (; cur < array_end; ++cur) {
3749                         char *key = (char*)cur->key;
3750
3751                         if (!key || key == tombstone)
3752                                 continue;
3753
3754                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3755                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3756                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3757
3758                         if (!sgen_is_object_alive_for_current_gen (key)) {
3759                                 cur->key = tombstone;
3760                                 cur->value = NULL;
3761                                 continue;
3762                         }
3763                 }
3764                 prev = current;
3765                 current = current->next;
3766         }
3767 }
3768
3769 /*
3770 LOCKING: requires that the GC lock is held
3771
3772 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3773 */
3774 static int
3775 mark_ephemerons_in_range (ScanCopyContext ctx)
3776 {
3777         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3778         GrayQueue *queue = ctx.queue;
3779         int nothing_marked = 1;
3780         EphemeronLinkNode *current = ephemeron_list;
3781         MonoArray *array;
3782         Ephemeron *cur, *array_end;
3783         char *tombstone;
3784
3785         for (current = ephemeron_list; current; current = current->next) {
3786                 char *object = current->array;
3787                 SGEN_LOG (5, "Ephemeron array at %p", object);
3788
3789                 /*It has to be alive*/
3790                 if (!sgen_is_object_alive_for_current_gen (object)) {
3791                         SGEN_LOG (5, "\tnot reachable");
3792                         continue;
3793                 }
3794
3795                 copy_func ((void**)&object, queue);
3796
3797                 array = (MonoArray*)object;
3798                 cur = mono_array_addr (array, Ephemeron, 0);
3799                 array_end = cur + mono_array_length_fast (array);
3800                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3801
3802                 for (; cur < array_end; ++cur) {
3803                         char *key = cur->key;
3804
3805                         if (!key || key == tombstone)
3806                                 continue;
3807
3808                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3809                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3810                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3811
3812                         if (sgen_is_object_alive_for_current_gen (key)) {
3813                                 char *value = cur->value;
3814
3815                                 copy_func ((void**)&cur->key, queue);
3816                                 if (value) {
3817                                         if (!sgen_is_object_alive_for_current_gen (value))
3818                                                 nothing_marked = 0;
3819                                         copy_func ((void**)&cur->value, queue);
3820                                 }
3821                         }
3822                 }
3823         }
3824
3825         SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3826         return nothing_marked;
3827 }
3828
3829 int
3830 mono_gc_invoke_finalizers (void)
3831 {
3832         FinalizeReadyEntry *entry = NULL;
3833         gboolean entry_is_critical = FALSE;
3834         int count = 0;
3835         void *obj;
3836         /* FIXME: batch to reduce lock contention */
3837         while (fin_ready_list || critical_fin_list) {
3838                 LOCK_GC;
3839
3840                 if (entry) {
3841                         FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3842
3843                         /* We have finalized entry in the last
3844                            interation, now we need to remove it from
3845                            the list. */
3846                         if (*list == entry)
3847                                 *list = entry->next;
3848                         else {
3849                                 FinalizeReadyEntry *e = *list;
3850                                 while (e->next != entry)
3851                                         e = e->next;
3852                                 e->next = entry->next;
3853                         }
3854                         sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3855                         entry = NULL;
3856                 }
3857
3858                 /* Now look for the first non-null entry. */
3859                 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3860                         ;
3861                 if (entry) {
3862                         entry_is_critical = FALSE;
3863                 } else {
3864                         entry_is_critical = TRUE;
3865                         for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3866                                 ;
3867                 }
3868
3869                 if (entry) {
3870                         g_assert (entry->object);
3871                         num_ready_finalizers--;
3872                         obj = entry->object;
3873                         entry->object = NULL;
3874                         SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3875                 }
3876
3877                 UNLOCK_GC;
3878
3879                 if (!entry)
3880                         break;
3881
3882                 g_assert (entry->object == NULL);
3883                 count++;
3884                 /* the object is on the stack so it is pinned */
3885                 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3886                 mono_gc_run_finalize (obj, NULL);
3887         }
3888         g_assert (!entry);
3889         return count;
3890 }
3891
3892 gboolean
3893 mono_gc_pending_finalizers (void)
3894 {
3895         return fin_ready_list || critical_fin_list;
3896 }
3897
3898 /*
3899  * ######################################################################
3900  * ########  registered roots support
3901  * ######################################################################
3902  */
3903
3904 /*
3905  * We do not coalesce roots.
3906  */
3907 static int
3908 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3909 {
3910         RootRecord new_root;
3911         int i;
3912         LOCK_GC;
3913         for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3914                 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3915                 /* we allow changing the size and the descriptor (for thread statics etc) */
3916                 if (root) {
3917                         size_t old_size = root->end_root - start;
3918                         root->end_root = start + size;
3919                         g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3920                                           ((root->root_desc == 0) && (descr == NULL)));
3921                         root->root_desc = (mword)descr;
3922                         roots_size += size;
3923                         roots_size -= old_size;
3924                         UNLOCK_GC;
3925                         return TRUE;
3926                 }
3927         }
3928
3929         new_root.end_root = start + size;
3930         new_root.root_desc = (mword)descr;
3931
3932         sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3933         roots_size += size;
3934
3935         SGEN_LOG (3, "Added root for range: %p-%p, descr: %p  (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3936
3937         UNLOCK_GC;
3938         return TRUE;
3939 }
3940
3941 int
3942 mono_gc_register_root (char *start, size_t size, void *descr)
3943 {
3944         return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3945 }
3946
3947 int
3948 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3949 {
3950         return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3951 }
3952
3953 void
3954 mono_gc_deregister_root (char* addr)
3955 {
3956         int root_type;
3957         RootRecord root;
3958
3959         LOCK_GC;
3960         for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3961                 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3962                         roots_size -= (root.end_root - addr);
3963         }
3964         UNLOCK_GC;
3965 }
3966
3967 /*
3968  * ######################################################################
3969  * ########  Thread handling (stop/start code)
3970  * ######################################################################
3971  */
3972
3973 unsigned int sgen_global_stop_count = 0;
3974
3975 void
3976 sgen_fill_thread_info_for_suspend (SgenThreadInfo *info)
3977 {
3978         if (remset.fill_thread_info_for_suspend)
3979                 remset.fill_thread_info_for_suspend (info);
3980 }
3981
3982 int
3983 sgen_get_current_collection_generation (void)
3984 {
3985         return current_collection_generation;
3986 }
3987
3988 void
3989 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3990 {
3991         gc_callbacks = *callbacks;
3992 }
3993
3994 MonoGCCallbacks *
3995 mono_gc_get_gc_callbacks ()
3996 {
3997         return &gc_callbacks;
3998 }
3999
4000 /* Variables holding start/end nursery so it won't have to be passed at every call */
4001 static void *scan_area_arg_start, *scan_area_arg_end;
4002
4003 void
4004 mono_gc_conservatively_scan_area (void *start, void *end)
4005 {
4006         conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
4007 }
4008
4009 void*
4010 mono_gc_scan_object (void *obj)
4011 {
4012         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
4013         current_object_ops.copy_or_mark_object (&obj, data->queue);
4014         return obj;
4015 }
4016
4017 /*
4018  * Mark from thread stacks and registers.
4019  */
4020 static void
4021 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
4022 {
4023         SgenThreadInfo *info;
4024
4025         scan_area_arg_start = start_nursery;
4026         scan_area_arg_end = end_nursery;
4027
4028         FOREACH_THREAD (info) {
4029                 if (info->skip) {
4030                         SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
4031                         continue;
4032                 }
4033                 if (info->gc_disabled) {
4034                         SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
4035                         continue;
4036                 }
4037
4038                 if (!info->joined_stw) {
4039                         SGEN_LOG (3, "Skipping thread not seen in STW %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
4040                         continue;
4041                 }
4042
4043                 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
4044                 if (!info->thread_is_dying) {
4045                         if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
4046                                 UserCopyOrMarkData data = { NULL, queue };
4047                                 set_user_copy_or_mark_data (&data);
4048                                 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
4049                                 set_user_copy_or_mark_data (NULL);
4050                         } else if (!precise) {
4051                                 if (!conservative_stack_mark) {
4052                                         fprintf (stderr, "Precise stack mark not supported - disabling.\n");
4053                                         conservative_stack_mark = TRUE;
4054                                 }
4055                                 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
4056                         }
4057                 }
4058
4059                 if (!info->thread_is_dying && !precise) {
4060 #ifdef USE_MONO_CTX
4061                         conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
4062                                 start_nursery, end_nursery, PIN_TYPE_STACK);
4063 #else
4064                         conservatively_pin_objects_from (&info->regs, &info->regs + ARCH_NUM_REGS,
4065                                         start_nursery, end_nursery, PIN_TYPE_STACK);
4066 #endif
4067                 }
4068         } END_FOREACH_THREAD
4069 }
4070
4071 static gboolean
4072 ptr_on_stack (void *ptr)
4073 {
4074         gpointer stack_start = &stack_start;
4075         SgenThreadInfo *info = mono_thread_info_current ();
4076
4077         if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
4078                 return TRUE;
4079         return FALSE;
4080 }
4081
4082 static void*
4083 sgen_thread_register (SgenThreadInfo* info, void *addr)
4084 {
4085 #ifndef HAVE_KW_THREAD
4086         SgenThreadInfo *__thread_info__ = info;
4087 #endif
4088
4089         LOCK_GC;
4090 #ifndef HAVE_KW_THREAD
4091         info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
4092
4093         g_assert (!mono_native_tls_get_value (thread_info_key));
4094         mono_native_tls_set_value (thread_info_key, info);
4095 #else
4096         sgen_thread_info = info;
4097 #endif
4098
4099 #if !defined(__MACH__)
4100         info->stop_count = -1;
4101         info->signal = 0;
4102 #endif
4103         info->skip = 0;
4104         info->joined_stw = FALSE;
4105         info->doing_handshake = FALSE;
4106         info->thread_is_dying = FALSE;
4107         info->stack_start = NULL;
4108         info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
4109         info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
4110         info->stopped_ip = NULL;
4111         info->stopped_domain = NULL;
4112 #ifdef USE_MONO_CTX
4113         memset (&info->ctx, 0, sizeof (MonoContext));
4114 #else
4115         memset (&info->regs, 0, sizeof (info->regs));
4116 #endif
4117
4118         sgen_init_tlab_info (info);
4119
4120         binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
4121
4122 #ifdef HAVE_KW_THREAD
4123         store_remset_buffer_index_addr = &store_remset_buffer_index;
4124 #endif
4125
4126         /* try to get it with attributes first */
4127 #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
4128         {
4129                 size_t size;
4130                 void *sstart;
4131                 pthread_attr_t attr;
4132                 pthread_getattr_np (pthread_self (), &attr);
4133                 pthread_attr_getstack (&attr, &sstart, &size);
4134                 info->stack_start_limit = sstart;
4135                 info->stack_end = (char*)sstart + size;
4136                 pthread_attr_destroy (&attr);
4137         }
4138 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
4139                  info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
4140                  info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
4141 #else
4142         {
4143                 /* FIXME: we assume the stack grows down */
4144                 gsize stack_bottom = (gsize)addr;
4145                 stack_bottom += 4095;
4146                 stack_bottom &= ~4095;
4147                 info->stack_end = (char*)stack_bottom;
4148         }
4149 #endif
4150
4151 #ifdef HAVE_KW_THREAD
4152         stack_end = info->stack_end;
4153 #endif
4154
4155         if (remset.register_thread)
4156                 remset.register_thread (info);
4157
4158         SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
4159
4160         if (gc_callbacks.thread_attach_func)
4161                 info->runtime_data = gc_callbacks.thread_attach_func ();
4162
4163         UNLOCK_GC;
4164         return info;
4165 }
4166
4167 static void
4168 sgen_wbarrier_cleanup_thread (SgenThreadInfo *p)
4169 {
4170         if (remset.cleanup_thread)
4171                 remset.cleanup_thread (p);
4172 }
4173
4174 static void
4175 sgen_thread_unregister (SgenThreadInfo *p)
4176 {
4177         /* If a delegate is passed to native code and invoked on a thread we dont
4178          * know about, the jit will register it with mono_jit_thread_attach, but
4179          * we have no way of knowing when that thread goes away.  SGen has a TSD
4180          * so we assume that if the domain is still registered, we can detach
4181          * the thread
4182          */
4183         if (mono_domain_get ())
4184                 mono_thread_detach (mono_thread_current ());
4185
4186         p->thread_is_dying = TRUE;
4187
4188         /*
4189         There is a race condition between a thread finishing executing and been removed
4190         from the GC thread set.
4191         This happens on posix systems when TLS data is been cleaned-up, libpthread will
4192         set the thread_info slot to NULL before calling the cleanup function. This
4193         opens a window in which the thread is registered but has a NULL TLS.
4194
4195         The suspend signal handler needs TLS data to know where to store thread state
4196         data or otherwise it will simply ignore the thread.
4197
4198         This solution works because the thread doing STW will wait until all threads been
4199         suspended handshake back, so there is no race between the doing_hankshake test
4200         and the suspend_thread call.
4201
4202         This is not required on systems that do synchronous STW as those can deal with
4203         the above race at suspend time.
4204
4205         FIXME: I believe we could avoid this by using mono_thread_info_lookup when
4206         mono_thread_info_current returns NULL. Or fix mono_thread_info_lookup to do so.
4207         */
4208 #if (defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED) || !defined(HAVE_PTHREAD_KILL)
4209         LOCK_GC;
4210 #else
4211         while (!TRYLOCK_GC) {
4212                 if (!sgen_park_current_thread_if_doing_handshake (p))
4213                         g_usleep (50);
4214         }
4215         MONO_GC_LOCKED ();
4216 #endif
4217
4218         binary_protocol_thread_unregister ((gpointer)mono_thread_info_get_tid (p));
4219         SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)mono_thread_info_get_tid (p));
4220
4221         if (gc_callbacks.thread_detach_func) {
4222                 gc_callbacks.thread_detach_func (p->runtime_data);
4223                 p->runtime_data = NULL;
4224         }
4225         sgen_wbarrier_cleanup_thread (p);
4226
4227         mono_threads_unregister_current_thread (p);
4228         UNLOCK_GC;
4229 }
4230
4231
4232 static void
4233 sgen_thread_attach (SgenThreadInfo *info)
4234 {
4235         LOCK_GC;
4236         /*this is odd, can we get attached before the gc is inited?*/
4237         init_stats ();
4238         UNLOCK_GC;
4239
4240         if (gc_callbacks.thread_attach_func && !info->runtime_data)
4241                 info->runtime_data = gc_callbacks.thread_attach_func ();
4242 }
4243 gboolean
4244 mono_gc_register_thread (void *baseptr)
4245 {
4246         return mono_thread_info_attach (baseptr) != NULL;
4247 }
4248
4249 /*
4250  * mono_gc_set_stack_end:
4251  *
4252  *   Set the end of the current threads stack to STACK_END. The stack space between
4253  * STACK_END and the real end of the threads stack will not be scanned during collections.
4254  */
4255 void
4256 mono_gc_set_stack_end (void *stack_end)
4257 {
4258         SgenThreadInfo *info;
4259
4260         LOCK_GC;
4261         info = mono_thread_info_current ();
4262         if (info) {
4263                 g_assert (stack_end < info->stack_end);
4264                 info->stack_end = stack_end;
4265         }
4266         UNLOCK_GC;
4267 }
4268
4269 #if USE_PTHREAD_INTERCEPT
4270
4271
4272 int
4273 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
4274 {
4275         return pthread_create (new_thread, attr, start_routine, arg);
4276 }
4277
4278 int
4279 mono_gc_pthread_join (pthread_t thread, void **retval)
4280 {
4281         return pthread_join (thread, retval);
4282 }
4283
4284 int
4285 mono_gc_pthread_detach (pthread_t thread)
4286 {
4287         return pthread_detach (thread);
4288 }
4289
4290 void
4291 mono_gc_pthread_exit (void *retval)
4292 {
4293         pthread_exit (retval);
4294 }
4295
4296 #endif /* USE_PTHREAD_INTERCEPT */
4297
4298 /*
4299  * ######################################################################
4300  * ########  Write barriers
4301  * ######################################################################
4302  */
4303
4304 /*
4305  * Note: the write barriers first do the needed GC work and then do the actual store:
4306  * this way the value is visible to the conservative GC scan after the write barrier
4307  * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
4308  * the conservative scan, otherwise by the remembered set scan.
4309  */
4310 void
4311 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
4312 {
4313         HEAVY_STAT (++stat_wbarrier_set_field);
4314         if (ptr_in_nursery (field_ptr)) {
4315                 *(void**)field_ptr = value;
4316                 return;
4317         }
4318         SGEN_LOG (8, "Adding remset at %p", field_ptr);
4319         if (value)
4320                 binary_protocol_wbarrier (field_ptr, value, value->vtable);
4321
4322         remset.wbarrier_set_field (obj, field_ptr, value);
4323 }
4324
4325 void
4326 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
4327 {
4328         HEAVY_STAT (++stat_wbarrier_set_arrayref);
4329         if (ptr_in_nursery (slot_ptr)) {
4330                 *(void**)slot_ptr = value;
4331                 return;
4332         }
4333         SGEN_LOG (8, "Adding remset at %p", slot_ptr);
4334         if (value)
4335                 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
4336
4337         remset.wbarrier_set_arrayref (arr, slot_ptr, value);
4338 }
4339
4340 void
4341 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
4342 {
4343         HEAVY_STAT (++stat_wbarrier_arrayref_copy);
4344         /*This check can be done without taking a lock since dest_ptr array is pinned*/
4345         if (ptr_in_nursery (dest_ptr) || count <= 0) {
4346                 mono_gc_memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
4347                 return;
4348         }
4349
4350 #ifdef SGEN_BINARY_PROTOCOL
4351         {
4352                 int i;
4353                 for (i = 0; i < count; ++i) {
4354                         gpointer dest = (gpointer*)dest_ptr + i;
4355                         gpointer obj = *((gpointer*)src_ptr + i);
4356                         if (obj)
4357                                 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
4358                 }
4359         }
4360 #endif
4361
4362         remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
4363 }
4364
4365 static char *found_obj;
4366
4367 static void
4368 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4369 {
4370         char *ptr = user_data;
4371
4372         if (ptr >= obj && ptr < obj + size) {
4373                 g_assert (!found_obj);
4374                 found_obj = obj;
4375         }
4376 }
4377
4378 /* for use in the debugger */
4379 char* find_object_for_ptr (char *ptr);
4380 char*
4381 find_object_for_ptr (char *ptr)
4382 {
4383         if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4384                 found_obj = NULL;
4385                 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4386                                 find_object_for_ptr_callback, ptr, TRUE);
4387                 if (found_obj)
4388                         return found_obj;
4389         }
4390
4391         found_obj = NULL;
4392         sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4393         if (found_obj)
4394                 return found_obj;
4395
4396         /*
4397          * Very inefficient, but this is debugging code, supposed to
4398          * be called from gdb, so we don't care.
4399          */
4400         found_obj = NULL;
4401         major_collector.iterate_objects (TRUE, TRUE, find_object_for_ptr_callback, ptr);
4402         return found_obj;
4403 }
4404
4405 void
4406 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4407 {
4408         gpointer obj;
4409
4410         HEAVY_STAT (++stat_wbarrier_generic_store);
4411
4412 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4413         /* FIXME: ptr_in_heap must be called with the GC lock held */
4414         if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4415                 char *start = find_object_for_ptr (ptr);
4416                 MonoObject *value = *(MonoObject**)ptr;
4417                 LOCK_GC;
4418                 g_assert (start);
4419                 if (start) {
4420                         MonoObject *obj = (MonoObject*)start;
4421                         if (obj->vtable->domain != value->vtable->domain)
4422                                 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4423                 }
4424                 UNLOCK_GC;
4425         }
4426 #endif
4427
4428         obj = *(gpointer*)ptr;
4429         if (obj)
4430                 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4431
4432         if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4433                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4434                 return;
4435         }
4436
4437         /*
4438          * We need to record old->old pointer locations for the
4439          * concurrent collector.
4440          */
4441         if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4442                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4443                 return;
4444         }
4445
4446         SGEN_LOG (8, "Adding remset at %p", ptr);
4447
4448         remset.wbarrier_generic_nostore (ptr);
4449 }
4450
4451 void
4452 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4453 {
4454         SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4455         *(void**)ptr = value;
4456         if (ptr_in_nursery (value))
4457                 mono_gc_wbarrier_generic_nostore (ptr);
4458         sgen_dummy_use (value);
4459 }
4460
4461 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4462 {
4463         mword *dest = _dest;
4464         mword *src = _src;
4465
4466         while (size) {
4467                 if (bitmap & 0x1)
4468                         mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4469                 else
4470                         *dest = *src;
4471                 ++src;
4472                 ++dest;
4473                 size -= SIZEOF_VOID_P;
4474                 bitmap >>= 1;
4475         }
4476 }
4477
4478 #ifdef SGEN_BINARY_PROTOCOL
4479 #undef HANDLE_PTR
4480 #define HANDLE_PTR(ptr,obj) do {                                        \
4481                 gpointer o = *(gpointer*)(ptr);                         \
4482                 if ((o)) {                                              \
4483                         gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4484                         binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4485                 }                                                       \
4486         } while (0)
4487
4488 static void
4489 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4490 {
4491 #define SCAN_OBJECT_NOVTABLE
4492 #include "sgen-scan-object.h"
4493 }
4494 #endif
4495
4496 void
4497 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4498 {
4499         HEAVY_STAT (++stat_wbarrier_value_copy);
4500         g_assert (klass->valuetype);
4501
4502         SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4503
4504         if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4505                 size_t element_size = mono_class_value_size (klass, NULL);
4506                 size_t size = count * element_size;
4507                 mono_gc_memmove (dest, src, size);
4508                 return;
4509         }
4510
4511 #ifdef SGEN_BINARY_PROTOCOL
4512         {
4513                 size_t element_size = mono_class_value_size (klass, NULL);
4514                 int i;
4515                 for (i = 0; i < count; ++i) {
4516                         scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4517                                         (char*)src + i * element_size - sizeof (MonoObject),
4518                                         (mword) klass->gc_descr);
4519                 }
4520         }
4521 #endif
4522
4523         remset.wbarrier_value_copy (dest, src, count, klass);
4524 }
4525
4526 /**
4527  * mono_gc_wbarrier_object_copy:
4528  *
4529  * Write barrier to call when obj is the result of a clone or copy of an object.
4530  */
4531 void
4532 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4533 {
4534         int size;
4535
4536         HEAVY_STAT (++stat_wbarrier_object_copy);
4537
4538         if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4539                 size = mono_object_class (obj)->instance_size;
4540                 mono_gc_memmove ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4541                                 size - sizeof (MonoObject));
4542                 return;
4543         }
4544
4545 #ifdef SGEN_BINARY_PROTOCOL
4546         scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4547 #endif
4548
4549         remset.wbarrier_object_copy (obj, src);
4550 }
4551
4552
4553 /*
4554  * ######################################################################
4555  * ########  Other mono public interface functions.
4556  * ######################################################################
4557  */
4558
4559 #define REFS_SIZE 128
4560 typedef struct {
4561         void *data;
4562         MonoGCReferences callback;
4563         int flags;
4564         int count;
4565         int called;
4566         MonoObject *refs [REFS_SIZE];
4567         uintptr_t offsets [REFS_SIZE];
4568 } HeapWalkInfo;
4569
4570 #undef HANDLE_PTR
4571 #define HANDLE_PTR(ptr,obj)     do {    \
4572                 if (*(ptr)) {   \
4573                         if (hwi->count == REFS_SIZE) {  \
4574                                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);    \
4575                                 hwi->count = 0; \
4576                                 hwi->called = 1;        \
4577                         }       \
4578                         hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start;  \
4579                         hwi->refs [hwi->count++] = *(ptr);      \
4580                 }       \
4581         } while (0)
4582
4583 static void
4584 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4585 {
4586 #include "sgen-scan-object.h"
4587 }
4588
4589 static void
4590 walk_references (char *start, size_t size, void *data)
4591 {
4592         HeapWalkInfo *hwi = data;
4593         hwi->called = 0;
4594         hwi->count = 0;
4595         collect_references (hwi, start, size);
4596         if (hwi->count || !hwi->called)
4597                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4598 }
4599
4600 /**
4601  * mono_gc_walk_heap:
4602  * @flags: flags for future use
4603  * @callback: a function pointer called for each object in the heap
4604  * @data: a user data pointer that is passed to callback
4605  *
4606  * This function can be used to iterate over all the live objects in the heap:
4607  * for each object, @callback is invoked, providing info about the object's
4608  * location in memory, its class, its size and the objects it references.
4609  * For each referenced object it's offset from the object address is
4610  * reported in the offsets array.
4611  * The object references may be buffered, so the callback may be invoked
4612  * multiple times for the same object: in all but the first call, the size
4613  * argument will be zero.
4614  * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4615  * profiler event handler.
4616  *
4617  * Returns: a non-zero value if the GC doesn't support heap walking
4618  */
4619 int
4620 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4621 {
4622         HeapWalkInfo hwi;
4623
4624         hwi.flags = flags;
4625         hwi.callback = callback;
4626         hwi.data = data;
4627
4628         sgen_clear_nursery_fragments ();
4629         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4630
4631         major_collector.iterate_objects (TRUE, TRUE, walk_references, &hwi);
4632         sgen_los_iterate_objects (walk_references, &hwi);
4633
4634         return 0;
4635 }
4636
4637 void
4638 mono_gc_collect (int generation)
4639 {
4640         LOCK_GC;
4641         if (generation > 1)
4642                 generation = 1;
4643         sgen_perform_collection (0, generation, "user request", TRUE);
4644         UNLOCK_GC;
4645 }
4646
4647 int
4648 mono_gc_max_generation (void)
4649 {
4650         return 1;
4651 }
4652
4653 int
4654 mono_gc_collection_count (int generation)
4655 {
4656         if (generation == 0)
4657                 return stat_minor_gcs;
4658         return stat_major_gcs;
4659 }
4660
4661 int64_t
4662 mono_gc_get_used_size (void)
4663 {
4664         gint64 tot = 0;
4665         LOCK_GC;
4666         tot = los_memory_usage;
4667         tot += nursery_section->next_data - nursery_section->data;
4668         tot += major_collector.get_used_size ();
4669         /* FIXME: account for pinned objects */
4670         UNLOCK_GC;
4671         return tot;
4672 }
4673
4674 void
4675 mono_gc_disable (void)
4676 {
4677         LOCK_GC;
4678         gc_disabled++;
4679         UNLOCK_GC;
4680 }
4681
4682 void
4683 mono_gc_enable (void)
4684 {
4685         LOCK_GC;
4686         gc_disabled--;
4687         UNLOCK_GC;
4688 }
4689
4690 int
4691 mono_gc_get_los_limit (void)
4692 {
4693         return MAX_SMALL_OBJ_SIZE;
4694 }
4695
4696 gboolean
4697 mono_gc_user_markers_supported (void)
4698 {
4699         return TRUE;
4700 }
4701
4702 gboolean
4703 mono_object_is_alive (MonoObject* o)
4704 {
4705         return TRUE;
4706 }
4707
4708 int
4709 mono_gc_get_generation (MonoObject *obj)
4710 {
4711         if (ptr_in_nursery (obj))
4712                 return 0;
4713         return 1;
4714 }
4715
4716 void
4717 mono_gc_enable_events (void)
4718 {
4719 }
4720
4721 void
4722 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4723 {
4724         sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4725 }
4726
4727 void
4728 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4729 {
4730         sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4731 }
4732
4733 MonoObject*
4734 mono_gc_weak_link_get (void **link_addr)
4735 {
4736         /*
4737          * We must only load *link_addr once because it might change
4738          * under our feet, and REVEAL_POINTER (NULL) results in an
4739          * invalid reference.
4740          */
4741         void *ptr = *link_addr;
4742         if (!ptr)
4743                 return NULL;
4744
4745         /*
4746          * During the second bridge processing step the world is
4747          * running again.  That step processes all weak links once
4748          * more to null those that refer to dead objects.  Before that
4749          * is completed, those links must not be followed, so we
4750          * conservatively wait for bridge processing when any weak
4751          * link is dereferenced.
4752          */
4753         if (G_UNLIKELY (bridge_processing_in_progress))
4754                 mono_gc_wait_for_bridge_processing ();
4755
4756         return (MonoObject*) REVEAL_POINTER (ptr);
4757 }
4758
4759 gboolean
4760 mono_gc_ephemeron_array_add (MonoObject *obj)
4761 {
4762         EphemeronLinkNode *node;
4763
4764         LOCK_GC;
4765
4766         node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4767         if (!node) {
4768                 UNLOCK_GC;
4769                 return FALSE;
4770         }
4771         node->array = (char*)obj;
4772         node->next = ephemeron_list;
4773         ephemeron_list = node;
4774
4775         SGEN_LOG (5, "Registered ephemeron array %p", obj);
4776
4777         UNLOCK_GC;
4778         return TRUE;
4779 }
4780
4781 void*
4782 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4783 {
4784         void *result;
4785         LOCK_INTERRUPTION;
4786         result = func (data);
4787         UNLOCK_INTERRUPTION;
4788         return result;
4789 }
4790
4791 gboolean
4792 mono_gc_is_gc_thread (void)
4793 {
4794         gboolean result;
4795         LOCK_GC;
4796         result = mono_thread_info_current () != NULL;
4797         UNLOCK_GC;
4798         return result;
4799 }
4800
4801 static gboolean
4802 is_critical_method (MonoMethod *method)
4803 {
4804         return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4805 }
4806
4807 void
4808 mono_gc_base_init (void)
4809 {
4810         MonoThreadInfoCallbacks cb;
4811         char *env;
4812         char **opts, **ptr;
4813         char *major_collector_opt = NULL;
4814         char *minor_collector_opt = NULL;
4815         glong max_heap = 0;
4816         glong soft_limit = 0;
4817         int num_workers;
4818         int result;
4819         int dummy;
4820         gboolean debug_print_allowance = FALSE;
4821         double allowance_ratio = 0, save_target = 0;
4822         gboolean have_split_nursery = FALSE;
4823         gboolean cement_enabled = TRUE;
4824
4825         do {
4826                 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4827                 switch (result) {
4828                 case 1:
4829                         /* already inited */
4830                         return;
4831                 case -1:
4832                         /* being inited by another thread */
4833                         g_usleep (1000);
4834                         break;
4835                 case 0:
4836                         /* we will init it */
4837                         break;
4838                 default:
4839                         g_assert_not_reached ();
4840                 }
4841         } while (result != 0);
4842
4843         LOCK_INIT (gc_mutex);
4844
4845         pagesize = mono_pagesize ();
4846         gc_debug_file = stderr;
4847
4848         cb.thread_register = sgen_thread_register;
4849         cb.thread_unregister = sgen_thread_unregister;
4850         cb.thread_attach = sgen_thread_attach;
4851         cb.mono_method_is_critical = (gpointer)is_critical_method;
4852 #ifndef HOST_WIN32
4853         cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4854 #endif
4855
4856         mono_threads_init (&cb, sizeof (SgenThreadInfo));
4857
4858         LOCK_INIT (sgen_interruption_mutex);
4859         LOCK_INIT (pin_queue_mutex);
4860
4861         init_user_copy_or_mark_key ();
4862
4863         if ((env = getenv ("MONO_GC_PARAMS"))) {
4864                 opts = g_strsplit (env, ",", -1);
4865                 for (ptr = opts; *ptr; ++ptr) {
4866                         char *opt = *ptr;
4867                         if (g_str_has_prefix (opt, "major=")) {
4868                                 opt = strchr (opt, '=') + 1;
4869                                 major_collector_opt = g_strdup (opt);
4870                         } else if (g_str_has_prefix (opt, "minor=")) {
4871                                 opt = strchr (opt, '=') + 1;
4872                                 minor_collector_opt = g_strdup (opt);
4873                         }
4874                 }
4875         } else {
4876                 opts = NULL;
4877         }
4878
4879         init_stats ();
4880         sgen_init_internal_allocator ();
4881         sgen_init_nursery_allocator ();
4882
4883         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4884         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4885         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4886         g_assert (sizeof (GenericStoreRememberedSet) == sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
4887         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_STORE_REMSET, sizeof (GenericStoreRememberedSet));
4888         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4889
4890 #ifndef HAVE_KW_THREAD
4891         mono_native_tls_alloc (&thread_info_key, NULL);
4892 #endif
4893
4894         /*
4895          * This needs to happen before any internal allocations because
4896          * it inits the small id which is required for hazard pointer
4897          * operations.
4898          */
4899         sgen_os_init ();
4900
4901         mono_thread_info_attach (&dummy);
4902
4903         if (!minor_collector_opt) {
4904                 sgen_simple_nursery_init (&sgen_minor_collector);
4905         } else {
4906                 if (!strcmp (minor_collector_opt, "simple")) {
4907                         sgen_simple_nursery_init (&sgen_minor_collector);
4908                 } else if (!strcmp (minor_collector_opt, "split")) {
4909                         sgen_split_nursery_init (&sgen_minor_collector);
4910                         have_split_nursery = TRUE;
4911                 } else {
4912                         fprintf (stderr, "Unknown minor collector `%s'.\n", minor_collector_opt);
4913                         exit (1);
4914                 }
4915         }
4916
4917         if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4918                 sgen_marksweep_init (&major_collector);
4919         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed")) {
4920                 sgen_marksweep_fixed_init (&major_collector);
4921         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-par")) {
4922                 sgen_marksweep_par_init (&major_collector);
4923         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) {
4924                 sgen_marksweep_fixed_par_init (&major_collector);
4925         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4926                 sgen_marksweep_conc_init (&major_collector);
4927         } else {
4928                 fprintf (stderr, "Unknown major collector `%s'.\n", major_collector_opt);
4929                 exit (1);
4930         }
4931
4932 #ifdef SGEN_HAVE_CARDTABLE
4933         use_cardtable = major_collector.supports_cardtable;
4934 #else
4935         use_cardtable = FALSE;
4936 #endif
4937
4938         num_workers = mono_cpu_count ();
4939         g_assert (num_workers > 0);
4940         if (num_workers > 16)
4941                 num_workers = 16;
4942
4943         ///* Keep this the default for now */
4944         /* Precise marking is broken on all supported targets. Disable until fixed. */
4945         conservative_stack_mark = TRUE;
4946
4947         sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4948
4949         if (opts) {
4950                 for (ptr = opts; *ptr; ++ptr) {
4951                         char *opt = *ptr;
4952                         if (g_str_has_prefix (opt, "major="))
4953                                 continue;
4954                         if (g_str_has_prefix (opt, "minor="))
4955                                 continue;
4956                         if (g_str_has_prefix (opt, "wbarrier=")) {
4957                                 opt = strchr (opt, '=') + 1;
4958                                 if (strcmp (opt, "remset") == 0) {
4959                                         if (major_collector.is_concurrent) {
4960                                                 fprintf (stderr, "The concurrent collector does not support the SSB write barrier.\n");
4961                                                 exit (1);
4962                                         }
4963                                         use_cardtable = FALSE;
4964                                 } else if (strcmp (opt, "cardtable") == 0) {
4965                                         if (!use_cardtable) {
4966                                                 if (major_collector.supports_cardtable)
4967                                                         fprintf (stderr, "The cardtable write barrier is not supported on this platform.\n");
4968                                                 else
4969                                                         fprintf (stderr, "The major collector does not support the cardtable write barrier.\n");
4970                                                 exit (1);
4971                                         }
4972                                 } else {
4973                                         fprintf (stderr, "wbarrier must either be `remset' or `cardtable'.");
4974                                         exit (1);
4975                                 }
4976                                 continue;
4977                         }
4978                         if (g_str_has_prefix (opt, "max-heap-size=")) {
4979                                 opt = strchr (opt, '=') + 1;
4980                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap)) {
4981                                         if ((max_heap & (mono_pagesize () - 1))) {
4982                                                 fprintf (stderr, "max-heap-size size must be a multiple of %d.\n", mono_pagesize ());
4983                                                 exit (1);
4984                                         }
4985                                 } else {
4986                                         fprintf (stderr, "max-heap-size must be an integer.\n");
4987                                         exit (1);
4988                                 }
4989                                 continue;
4990                         }
4991                         if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4992                                 opt = strchr (opt, '=') + 1;
4993                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4994                                         if (soft_limit <= 0) {
4995                                                 fprintf (stderr, "soft-heap-limit must be positive.\n");
4996                                                 exit (1);
4997                                         }
4998                                 } else {
4999                                         fprintf (stderr, "soft-heap-limit must be an integer.\n");
5000                                         exit (1);
5001                                 }
5002                                 continue;
5003                         }
5004                         if (g_str_has_prefix (opt, "workers=")) {
5005                                 long val;
5006                                 char *endptr;
5007                                 if (!major_collector.is_parallel) {
5008                                         fprintf (stderr, "The workers= option can only be used for parallel collectors.");
5009                                         exit (1);
5010                                 }
5011                                 opt = strchr (opt, '=') + 1;
5012                                 val = strtol (opt, &endptr, 10);
5013                                 if (!*opt || *endptr) {
5014                                         fprintf (stderr, "Cannot parse the workers= option value.");
5015                                         exit (1);
5016                                 }
5017                                 if (val <= 0 || val > 16) {
5018                                         fprintf (stderr, "The number of workers must be in the range 1 to 16.");
5019                                         exit (1);
5020                                 }
5021                                 num_workers = (int)val;
5022                                 continue;
5023                         }
5024                         if (g_str_has_prefix (opt, "stack-mark=")) {
5025                                 opt = strchr (opt, '=') + 1;
5026                                 if (!strcmp (opt, "precise")) {
5027                                         conservative_stack_mark = FALSE;
5028                                 } else if (!strcmp (opt, "conservative")) {
5029                                         conservative_stack_mark = TRUE;
5030                                 } else {
5031                                         fprintf (stderr, "Invalid value '%s' for stack-mark= option, possible values are: 'precise', 'conservative'.\n", opt);
5032                                         exit (1);
5033                                 }
5034                                 continue;
5035                         }
5036                         if (g_str_has_prefix (opt, "bridge=")) {
5037                                 opt = strchr (opt, '=') + 1;
5038                                 sgen_register_test_bridge_callbacks (g_strdup (opt));
5039                                 continue;
5040                         }
5041 #ifdef USER_CONFIG
5042                         if (g_str_has_prefix (opt, "nursery-size=")) {
5043                                 long val;
5044                                 opt = strchr (opt, '=') + 1;
5045                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
5046                                         sgen_nursery_size = val;
5047 #ifdef SGEN_ALIGN_NURSERY
5048                                         if ((val & (val - 1))) {
5049                                                 fprintf (stderr, "The nursery size must be a power of two.\n");
5050                                                 exit (1);
5051                                         }
5052
5053                                         if (val < SGEN_MAX_NURSERY_WASTE) {
5054                                                 fprintf (stderr, "The nursery size must be at least %d bytes.\n", SGEN_MAX_NURSERY_WASTE);
5055                                                 exit (1);
5056                                         }
5057
5058                                         sgen_nursery_bits = 0;
5059                                         while (1 << (++ sgen_nursery_bits) != sgen_nursery_size)
5060                                                 ;
5061 #endif
5062                                 } else {
5063                                         fprintf (stderr, "nursery-size must be an integer.\n");
5064                                         exit (1);
5065                                 }
5066                                 continue;
5067                         }
5068 #endif
5069                         if (g_str_has_prefix (opt, "save-target-ratio=")) {
5070                                 char *endptr;
5071                                 opt = strchr (opt, '=') + 1;
5072                                 save_target = strtod (opt, &endptr);
5073                                 if (endptr == opt) {
5074                                         fprintf (stderr, "save-target-ratio must be a number.");
5075                                         exit (1);
5076                                 }
5077                                 if (save_target < SGEN_MIN_SAVE_TARGET_RATIO || save_target > SGEN_MAX_SAVE_TARGET_RATIO) {
5078                                         fprintf (stderr, "save-target-ratio must be between %.2f - %.2f.", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
5079                                         exit (1);
5080                                 }
5081                                 continue;
5082                         }
5083                         if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
5084                                 char *endptr;
5085                                 opt = strchr (opt, '=') + 1;
5086
5087                                 allowance_ratio = strtod (opt, &endptr);
5088                                 if (endptr == opt) {
5089                                         fprintf (stderr, "save-target-ratio must be a number.");
5090                                         exit (1);
5091                                 }
5092                                 if (allowance_ratio < SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO || allowance_ratio > SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO) {
5093                                         fprintf (stderr, "default-allowance-ratio must be between %.2f - %.2f.", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO);
5094                                         exit (1);
5095                                 }
5096                                 continue;
5097                         }
5098
5099                         if (!strcmp (opt, "cementing")) {
5100                                 cement_enabled = TRUE;
5101                                 continue;
5102                         }
5103                         if (!strcmp (opt, "no-cementing")) {
5104                                 cement_enabled = FALSE;
5105                                 continue;
5106                         }
5107
5108                         if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
5109                                 continue;
5110
5111                         if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
5112                                 continue;
5113
5114                         fprintf (stderr, "MONO_GC_PARAMS must be a comma-delimited list of one or more of the following:\n");
5115                         fprintf (stderr, "  max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5116                         fprintf (stderr, "  soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
5117                         fprintf (stderr, "  nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5118                         fprintf (stderr, "  major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par', 'marksweep-fixed' or 'marksweep-fixed-par')\n");
5119                         fprintf (stderr, "  minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
5120                         fprintf (stderr, "  wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
5121                         fprintf (stderr, "  stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
5122                         fprintf (stderr, "  [no-]cementing\n");
5123                         if (major_collector.print_gc_param_usage)
5124                                 major_collector.print_gc_param_usage ();
5125                         if (sgen_minor_collector.print_gc_param_usage)
5126                                 sgen_minor_collector.print_gc_param_usage ();
5127                         fprintf (stderr, " Experimental options:\n");
5128                         fprintf (stderr, "  save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
5129                         fprintf (stderr, "  default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
5130                         exit (1);
5131                 }
5132                 g_strfreev (opts);
5133         }
5134
5135         if (major_collector.is_parallel)
5136                 sgen_workers_init (num_workers);
5137         else if (major_collector.is_concurrent)
5138                 sgen_workers_init (1);
5139
5140         if (major_collector_opt)
5141                 g_free (major_collector_opt);
5142
5143         if (minor_collector_opt)
5144                 g_free (minor_collector_opt);
5145
5146         alloc_nursery ();
5147
5148         sgen_cement_init (cement_enabled);
5149
5150         if ((env = getenv ("MONO_GC_DEBUG"))) {
5151                 opts = g_strsplit (env, ",", -1);
5152                 for (ptr = opts; ptr && *ptr; ptr ++) {
5153                         char *opt = *ptr;
5154                         if (opt [0] >= '0' && opt [0] <= '9') {
5155                                 gc_debug_level = atoi (opt);
5156                                 opt++;
5157                                 if (opt [0] == ':')
5158                                         opt++;
5159                                 if (opt [0]) {
5160 #ifdef HOST_WIN32
5161                                         char *rf = g_strdup_printf ("%s.%d", opt, GetCurrentProcessId ());
5162 #else
5163                                         char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
5164 #endif
5165                                         gc_debug_file = fopen (rf, "wb");
5166                                         if (!gc_debug_file)
5167                                                 gc_debug_file = stderr;
5168                                         g_free (rf);
5169                                 }
5170                         } else if (!strcmp (opt, "print-allowance")) {
5171                                 debug_print_allowance = TRUE;
5172                         } else if (!strcmp (opt, "print-pinning")) {
5173                                 do_pin_stats = TRUE;
5174                         } else if (!strcmp (opt, "verify-before-allocs")) {
5175                                 verify_before_allocs = 1;
5176                                 has_per_allocation_action = TRUE;
5177                         } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
5178                                 char *arg = strchr (opt, '=') + 1;
5179                                 verify_before_allocs = atoi (arg);
5180                                 has_per_allocation_action = TRUE;
5181                         } else if (!strcmp (opt, "collect-before-allocs")) {
5182                                 collect_before_allocs = 1;
5183                                 has_per_allocation_action = TRUE;
5184                         } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
5185                                 char *arg = strchr (opt, '=') + 1;
5186                                 has_per_allocation_action = TRUE;
5187                                 collect_before_allocs = atoi (arg);
5188                         } else if (!strcmp (opt, "verify-before-collections")) {
5189                                 whole_heap_check_before_collection = TRUE;
5190                         } else if (!strcmp (opt, "check-at-minor-collections")) {
5191                                 consistency_check_at_minor_collection = TRUE;
5192                                 nursery_clear_policy = CLEAR_AT_GC;
5193                         } else if (!strcmp (opt, "check-mark-bits")) {
5194                                 check_mark_bits_after_major_collection = TRUE;
5195                         } else if (!strcmp (opt, "check-nursery-pinned")) {
5196                                 check_nursery_objects_pinned = TRUE;
5197                         } else if (!strcmp (opt, "xdomain-checks")) {
5198                                 xdomain_checks = TRUE;
5199                         } else if (!strcmp (opt, "clear-at-gc")) {
5200                                 nursery_clear_policy = CLEAR_AT_GC;
5201                         } else if (!strcmp (opt, "clear-nursery-at-gc")) {
5202                                 nursery_clear_policy = CLEAR_AT_GC;
5203                         } else if (!strcmp (opt, "check-scan-starts")) {
5204                                 do_scan_starts_check = TRUE;
5205                         } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
5206                                 do_verify_nursery = TRUE;
5207                         } else if (!strcmp (opt, "check-concurrent")) {
5208                                 if (!major_collector.is_concurrent) {
5209                                         fprintf (stderr, "Error: check-concurrent only world with concurrent major collectors.\n");
5210                                         exit (1);
5211                                 }
5212                                 do_concurrent_checks = TRUE;
5213                         } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
5214                                 do_dump_nursery_content = TRUE;
5215                         } else if (!strcmp (opt, "no-managed-allocator")) {
5216                                 sgen_set_use_managed_allocator (FALSE);
5217                         } else if (!strcmp (opt, "disable-minor")) {
5218                                 disable_minor_collections = TRUE;
5219                         } else if (!strcmp (opt, "disable-major")) {
5220                                 disable_major_collections = TRUE;
5221                         } else if (g_str_has_prefix (opt, "heap-dump=")) {
5222                                 char *filename = strchr (opt, '=') + 1;
5223                                 nursery_clear_policy = CLEAR_AT_GC;
5224                                 heap_dump_file = fopen (filename, "w");
5225                                 if (heap_dump_file) {
5226                                         fprintf (heap_dump_file, "<sgen-dump>\n");
5227                                         do_pin_stats = TRUE;
5228                                 }
5229 #ifdef SGEN_BINARY_PROTOCOL
5230                         } else if (g_str_has_prefix (opt, "binary-protocol=")) {
5231                                 char *filename = strchr (opt, '=') + 1;
5232                                 binary_protocol_init (filename);
5233                                 if (use_cardtable)
5234                                         fprintf (stderr, "Warning: Cardtable write barriers will not be binary-protocolled.\n");
5235 #endif
5236                         } else {
5237                                 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
5238                                 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
5239                                 fprintf (stderr, "Valid options are:\n");
5240                                 fprintf (stderr, "  collect-before-allocs[=<n>]\n");
5241                                 fprintf (stderr, "  verify-before-allocs[=<n>]\n");
5242                                 fprintf (stderr, "  check-at-minor-collections\n");
5243                                 fprintf (stderr, "  check-mark-bits\n");
5244                                 fprintf (stderr, "  check-nursery-pinned\n");
5245                                 fprintf (stderr, "  verify-before-collections\n");
5246                                 fprintf (stderr, "  verify-nursery-at-minor-gc\n");
5247                                 fprintf (stderr, "  dump-nursery-at-minor-gc\n");
5248                                 fprintf (stderr, "  disable-minor\n");
5249                                 fprintf (stderr, "  disable-major\n");
5250                                 fprintf (stderr, "  xdomain-checks\n");
5251                                 fprintf (stderr, "  check-concurrent\n");
5252                                 fprintf (stderr, "  clear-at-gc\n");
5253                                 fprintf (stderr, "  clear-nursery-at-gc\n");
5254                                 fprintf (stderr, "  check-scan-starts\n");
5255                                 fprintf (stderr, "  no-managed-allocator\n");
5256                                 fprintf (stderr, "  print-allowance\n");
5257                                 fprintf (stderr, "  print-pinning\n");
5258                                 fprintf (stderr, "  heap-dump=<filename>\n");
5259 #ifdef SGEN_BINARY_PROTOCOL
5260                                 fprintf (stderr, "  binary-protocol=<filename>\n");
5261 #endif
5262                                 exit (1);
5263                         }
5264                 }
5265                 g_strfreev (opts);
5266         }
5267
5268         if (major_collector.is_parallel) {
5269                 if (heap_dump_file) {
5270                         fprintf (stderr, "Error: Cannot do heap dump with the parallel collector.\n");
5271                         exit (1);
5272                 }
5273                 if (do_pin_stats) {
5274                         fprintf (stderr, "Error: Cannot gather pinning statistics with the parallel collector.\n");
5275                         exit (1);
5276                 }
5277         }
5278
5279         if (major_collector.post_param_init)
5280                 major_collector.post_param_init (&major_collector);
5281
5282         sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
5283
5284         memset (&remset, 0, sizeof (remset));
5285
5286 #ifdef SGEN_HAVE_CARDTABLE
5287         if (use_cardtable)
5288                 sgen_card_table_init (&remset);
5289         else
5290 #endif
5291                 sgen_ssb_init (&remset);
5292
5293         if (remset.register_thread)
5294                 remset.register_thread (mono_thread_info_current ());
5295
5296         gc_initialized = 1;
5297 }
5298
5299 const char *
5300 mono_gc_get_gc_name (void)
5301 {
5302         return "sgen";
5303 }
5304
5305 static MonoMethod *write_barrier_method;
5306
5307 gboolean
5308 sgen_is_critical_method (MonoMethod *method)
5309 {
5310         return (method == write_barrier_method || sgen_is_managed_allocator (method));
5311 }
5312
5313 gboolean
5314 sgen_has_critical_method (void)
5315 {
5316         return write_barrier_method || sgen_has_managed_allocator ();
5317 }
5318
5319 static void
5320 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5321 {
5322         memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5323 #ifdef SGEN_ALIGN_NURSERY
5324         // if (ptr_in_nursery (ptr)) return;
5325         /*
5326          * Masking out the bits might be faster, but we would have to use 64 bit
5327          * immediates, which might be slower.
5328          */
5329         mono_mb_emit_ldarg (mb, 0);
5330         mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5331         mono_mb_emit_byte (mb, CEE_SHR_UN);
5332         mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5333         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5334
5335         if (!major_collector.is_concurrent) {
5336                 // if (!ptr_in_nursery (*ptr)) return;
5337                 mono_mb_emit_ldarg (mb, 0);
5338                 mono_mb_emit_byte (mb, CEE_LDIND_I);
5339                 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5340                 mono_mb_emit_byte (mb, CEE_SHR_UN);
5341                 mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5342                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5343         }
5344 #else
5345         int label_continue1, label_continue2;
5346         int dereferenced_var;
5347
5348         // if (ptr < (sgen_get_nursery_start ())) goto continue;
5349         mono_mb_emit_ldarg (mb, 0);
5350         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5351         label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5352
5353         // if (ptr >= sgen_get_nursery_end ())) goto continue;
5354         mono_mb_emit_ldarg (mb, 0);
5355         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5356         label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5357
5358         // Otherwise return
5359         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5360
5361         // continue:
5362         mono_mb_patch_branch (mb, label_continue_1);
5363         mono_mb_patch_branch (mb, label_continue_2);
5364
5365         // Dereference and store in local var
5366         dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5367         mono_mb_emit_ldarg (mb, 0);
5368         mono_mb_emit_byte (mb, CEE_LDIND_I);
5369         mono_mb_emit_stloc (mb, dereferenced_var);
5370
5371         if (!major_collector.is_concurrent) {
5372                 // if (*ptr < sgen_get_nursery_start ()) return;
5373                 mono_mb_emit_ldloc (mb, dereferenced_var);
5374                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5375                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5376
5377                 // if (*ptr >= sgen_get_nursery_end ()) return;
5378                 mono_mb_emit_ldloc (mb, dereferenced_var);
5379                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5380                 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5381         }
5382 #endif
5383 }
5384
5385 MonoMethod*
5386 mono_gc_get_write_barrier (void)
5387 {
5388         MonoMethod *res;
5389         MonoMethodBuilder *mb;
5390         MonoMethodSignature *sig;
5391 #ifdef MANAGED_WBARRIER
5392         int i, nursery_check_labels [3];
5393         int label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
5394         int buffer_var, buffer_index_var, dummy_var;
5395
5396 #ifdef HAVE_KW_THREAD
5397         int stack_end_offset = -1, store_remset_buffer_offset = -1;
5398         int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
5399
5400         MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5401         g_assert (stack_end_offset != -1);
5402         MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
5403         g_assert (store_remset_buffer_offset != -1);
5404         MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
5405         g_assert (store_remset_buffer_index_offset != -1);
5406         MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
5407         g_assert (store_remset_buffer_index_addr_offset != -1);
5408 #endif
5409 #endif
5410
5411         // FIXME: Maybe create a separate version for ctors (the branch would be
5412         // correctly predicted more times)
5413         if (write_barrier_method)
5414                 return write_barrier_method;
5415
5416         /* Create the IL version of mono_gc_barrier_generic_store () */
5417         sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5418         sig->ret = &mono_defaults.void_class->byval_arg;
5419         sig->params [0] = &mono_defaults.int_class->byval_arg;
5420
5421         mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5422
5423 #ifdef MANAGED_WBARRIER
5424         if (use_cardtable) {
5425                 emit_nursery_check (mb, nursery_check_labels);
5426                 /*
5427                 addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5428                 *addr = 1;
5429
5430                 sgen_cardtable:
5431                         LDC_PTR sgen_cardtable
5432
5433                 address >> CARD_BITS
5434                         LDARG_0
5435                         LDC_I4 CARD_BITS
5436                         SHR_UN
5437                 if (SGEN_HAVE_OVERLAPPING_CARDS) {
5438                         LDC_PTR card_table_mask
5439                         AND
5440                 }
5441                 AND
5442                 ldc_i4_1
5443                 stind_i1
5444                 */
5445                 mono_mb_emit_ptr (mb, sgen_cardtable);
5446                 mono_mb_emit_ldarg (mb, 0);
5447                 mono_mb_emit_icon (mb, CARD_BITS);
5448                 mono_mb_emit_byte (mb, CEE_SHR_UN);
5449 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5450                 mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5451                 mono_mb_emit_byte (mb, CEE_AND);
5452 #endif
5453                 mono_mb_emit_byte (mb, CEE_ADD);
5454                 mono_mb_emit_icon (mb, 1);
5455                 mono_mb_emit_byte (mb, CEE_STIND_I1);
5456
5457                 // return;
5458                 for (i = 0; i < 3; ++i) {
5459                         if (nursery_check_labels [i])
5460                                 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5461                 }
5462                 mono_mb_emit_byte (mb, CEE_RET);
5463         } else if (mono_runtime_has_tls_get ()) {
5464                 emit_nursery_check (mb, nursery_check_labels);
5465
5466                 // if (ptr >= stack_end) goto need_wb;
5467                 mono_mb_emit_ldarg (mb, 0);
5468                 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
5469                 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
5470
5471                 // if (ptr >= stack_start) return;
5472                 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5473                 mono_mb_emit_ldarg (mb, 0);
5474                 mono_mb_emit_ldloc_addr (mb, dummy_var);
5475                 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
5476
5477                 // need_wb:
5478                 mono_mb_patch_branch (mb, label_need_wb);
5479
5480                 // buffer = STORE_REMSET_BUFFER;
5481                 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5482                 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
5483                 mono_mb_emit_stloc (mb, buffer_var);
5484
5485                 // buffer_index = STORE_REMSET_BUFFER_INDEX;
5486                 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5487                 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
5488                 mono_mb_emit_stloc (mb, buffer_index_var);
5489
5490                 // if (buffer [buffer_index] == ptr) return;
5491                 mono_mb_emit_ldloc (mb, buffer_var);
5492                 mono_mb_emit_ldloc (mb, buffer_index_var);
5493                 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
5494                 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
5495                 mono_mb_emit_byte (mb, CEE_SHL);
5496                 mono_mb_emit_byte (mb, CEE_ADD);
5497                 mono_mb_emit_byte (mb, CEE_LDIND_I);
5498                 mono_mb_emit_ldarg (mb, 0);
5499                 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
5500
5501                 // ++buffer_index;
5502                 mono_mb_emit_ldloc (mb, buffer_index_var);
5503                 mono_mb_emit_icon (mb, 1);
5504                 mono_mb_emit_byte (mb, CEE_ADD);
5505                 mono_mb_emit_stloc (mb, buffer_index_var);
5506
5507                 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
5508                 mono_mb_emit_ldloc (mb, buffer_index_var);
5509                 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
5510                 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
5511
5512                 // buffer [buffer_index] = ptr;
5513                 mono_mb_emit_ldloc (mb, buffer_var);
5514                 mono_mb_emit_ldloc (mb, buffer_index_var);
5515                 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
5516                 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
5517                 mono_mb_emit_byte (mb, CEE_SHL);
5518                 mono_mb_emit_byte (mb, CEE_ADD);
5519                 mono_mb_emit_ldarg (mb, 0);
5520                 mono_mb_emit_byte (mb, CEE_STIND_I);
5521
5522                 // STORE_REMSET_BUFFER_INDEX = buffer_index;
5523                 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
5524                 mono_mb_emit_ldloc (mb, buffer_index_var);
5525                 mono_mb_emit_byte (mb, CEE_STIND_I);
5526
5527                 // return;
5528                 for (i = 0; i < 3; ++i) {
5529                         if (nursery_check_labels [i])
5530                                 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5531                 }
5532                 mono_mb_patch_branch (mb, label_no_wb_3);
5533                 mono_mb_patch_branch (mb, label_no_wb_4);
5534                 mono_mb_emit_byte (mb, CEE_RET);
5535
5536                 // slow path
5537                 mono_mb_patch_branch (mb, label_slow_path);
5538
5539                 mono_mb_emit_ldarg (mb, 0);
5540                 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5541                 mono_mb_emit_byte (mb, CEE_RET);
5542         } else
5543 #endif
5544         {
5545                 mono_mb_emit_ldarg (mb, 0);
5546                 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5547                 mono_mb_emit_byte (mb, CEE_RET);
5548         }
5549
5550         res = mono_mb_create_method (mb, sig, 16);
5551         mono_mb_free (mb);
5552
5553         mono_loader_lock ();
5554         if (write_barrier_method) {
5555                 /* Already created */
5556                 mono_free_method (res);
5557         } else {
5558                 /* double-checked locking */
5559                 mono_memory_barrier ();
5560                 write_barrier_method = res;
5561         }
5562         mono_loader_unlock ();
5563
5564         return write_barrier_method;
5565 }
5566
5567 char*
5568 mono_gc_get_description (void)
5569 {
5570         return g_strdup ("sgen");
5571 }
5572
5573 void
5574 mono_gc_set_desktop_mode (void)
5575 {
5576 }
5577
5578 gboolean
5579 mono_gc_is_moving (void)
5580 {
5581         return TRUE;
5582 }
5583
5584 gboolean
5585 mono_gc_is_disabled (void)
5586 {
5587         return FALSE;
5588 }
5589
5590 #ifdef HOST_WIN32
5591 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5592 {
5593         return TRUE;
5594 }
5595 #endif
5596
5597 NurseryClearPolicy
5598 sgen_get_nursery_clear_policy (void)
5599 {
5600         return nursery_clear_policy;
5601 }
5602
5603 MonoVTable*
5604 sgen_get_array_fill_vtable (void)
5605 {
5606         if (!array_fill_vtable) {
5607                 static MonoClass klass;
5608                 static MonoVTable vtable;
5609                 gsize bmap;
5610
5611                 MonoDomain *domain = mono_get_root_domain ();
5612                 g_assert (domain);
5613
5614                 klass.element_class = mono_defaults.byte_class;
5615                 klass.rank = 1;
5616                 klass.instance_size = sizeof (MonoArray);
5617                 klass.sizes.element_size = 1;
5618                 klass.name = "array_filler_type";
5619
5620                 vtable.klass = &klass;
5621                 bmap = 0;
5622                 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5623                 vtable.rank = 1;
5624
5625                 array_fill_vtable = &vtable;
5626         }
5627         return array_fill_vtable;
5628 }
5629
5630 void
5631 sgen_gc_lock (void)
5632 {
5633         LOCK_GC;
5634 }
5635
5636 void
5637 sgen_gc_unlock (void)
5638 {
5639         UNLOCK_GC;
5640 }
5641
5642 void
5643 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5644 {
5645         major_collector.iterate_live_block_ranges (callback);
5646 }
5647
5648 void
5649 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5650 {
5651         major_collector.scan_card_table (FALSE, queue);
5652 }
5653
5654 SgenMajorCollector*
5655 sgen_get_major_collector (void)
5656 {
5657         return &major_collector;
5658 }
5659
5660 void mono_gc_set_skip_thread (gboolean skip)
5661 {
5662         SgenThreadInfo *info = mono_thread_info_current ();
5663
5664         LOCK_GC;
5665         info->gc_disabled = skip;
5666         UNLOCK_GC;
5667 }
5668
5669 SgenRemeberedSet*
5670 sgen_get_remset (void)
5671 {
5672         return &remset;
5673 }
5674
5675 guint
5676 mono_gc_get_vtable_bits (MonoClass *class)
5677 {
5678         if (sgen_need_bridge_processing () && sgen_is_bridge_class (class))
5679                 return SGEN_GC_BIT_BRIDGE_OBJECT;
5680         return 0;
5681 }
5682
5683 void
5684 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5685 {
5686         // FIXME:
5687 }
5688
5689
5690 void
5691 sgen_check_whole_heap_stw (void)
5692 {
5693         sgen_stop_world (0);
5694         sgen_clear_nursery_fragments ();
5695         sgen_check_whole_heap (FALSE);
5696         sgen_restart_world (0, NULL);
5697 }
5698
5699 void
5700 sgen_gc_event_moves (void)
5701 {
5702         if (moved_objects_idx) {
5703                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5704                 moved_objects_idx = 0;
5705         }
5706 }
5707
5708 #endif /* HAVE_SGEN_GC */