mono/metadata/sgen-gc.c

   1 /*
   2  * sgen-gc.c: Simple generational GC.
   3  *
   4  * Author:
   5  *      Paolo Molaro (lupus@ximian.com)
   6  *  Rodrigo Kumpera (kumpera@gmail.com)
   7  *
   8  * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
   9  * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
  10  *
  11  * Thread start/stop adapted from Boehm's GC:
  12  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  13  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  14  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  15  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  16  * Copyright 2001-2003 Ximian, Inc
  17  * Copyright 2003-2010 Novell, Inc.
  18  * Copyright 2011 Xamarin, Inc.
  19  * Copyright (C) 2012 Xamarin Inc
  20  *
  21  * This library is free software; you can redistribute it and/or
  22  * modify it under the terms of the GNU Library General Public
  23  * License 2.0 as published by the Free Software Foundation;
  24  *
  25  * This library is distributed in the hope that it will be useful,
  26  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  28  * Library General Public License for more details.
  29  *
  30  * You should have received a copy of the GNU Library General Public
  31  * License 2.0 along with this library; if not, write to the Free
  32  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  33  *
  34  * Important: allocation provides always zeroed memory, having to do
  35  * a memset after allocation is deadly for performance.
  36  * Memory usage at startup is currently as follows:
  37  * 64 KB pinned space
  38  * 64 KB internal space
  39  * size of nursery
  40  * We should provide a small memory config with half the sizes
  41  *
  42  * We currently try to make as few mono assumptions as possible:
  43  * 1) 2-word header with no GC pointers in it (first vtable, second to store the
  44  *    forwarding ptr)
  45  * 2) gc descriptor is the second word in the vtable (first word in the class)
  46  * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
  47  * 4) there is a function to get an object's size and the number of
  48  *    elements in an array.
  49  * 5) we know the special way bounds are allocated for complex arrays
  50  * 6) we know about proxies and how to treat them when domains are unloaded
  51  *
  52  * Always try to keep stack usage to a minimum: no recursive behaviour
  53  * and no large stack allocs.
  54  *
  55  * General description.
  56  * Objects are initially allocated in a nursery using a fast bump-pointer technique.
  57  * When the nursery is full we start a nursery collection: this is performed with a
  58  * copying GC.
  59  * When the old generation is full we start a copying GC of the old generation as well:
  60  * this will be changed to mark&sweep with copying when fragmentation becomes to severe
  61  * in the future.  Maybe we'll even do both during the same collection like IMMIX.
  62  *
  63  * The things that complicate this description are:
  64  * *) pinned objects: we can't move them so we need to keep track of them
  65  * *) no precise info of the thread stacks and registers: we need to be able to
  66  *    quickly find the objects that may be referenced conservatively and pin them
  67  *    (this makes the first issues more important)
  68  * *) large objects are too expensive to be dealt with using copying GC: we handle them
  69  *    with mark/sweep during major collections
  70  * *) some objects need to not move even if they are small (interned strings, Type handles):
  71  *    we use mark/sweep for them, too: they are not allocated in the nursery, but inside
  72  *    PinnedChunks regions
  73  */
  74
  75 /*
  76  * TODO:
  77
  78  *) we could have a function pointer in MonoClass to implement
  79   customized write barriers for value types
  80
  81  *) investigate the stuff needed to advance a thread to a GC-safe
  82   point (single-stepping, read from unmapped memory etc) and implement it.
  83   This would enable us to inline allocations and write barriers, for example,
  84   or at least parts of them, like the write barrier checks.
  85   We may need this also for handling precise info on stacks, even simple things
  86   as having uninitialized data on the stack and having to wait for the prolog
  87   to zero it. Not an issue for the last frame that we scan conservatively.
  88   We could always not trust the value in the slots anyway.
  89
  90  *) modify the jit to save info about references in stack locations:
  91   this can be done just for locals as a start, so that at least
  92   part of the stack is handled precisely.
  93
  94  *) test/fix endianess issues
  95
  96  *) Implement a card table as the write barrier instead of remembered
  97     sets?  Card tables are not easy to implement with our current
  98     memory layout.  We have several different kinds of major heap
  99     objects: Small objects in regular blocks, small objects in pinned
 100     chunks and LOS objects.  If we just have a pointer we have no way
 101     to tell which kind of object it points into, therefore we cannot
 102     know where its card table is.  The least we have to do to make
 103     this happen is to get rid of write barriers for indirect stores.
 104     (See next item)
 105
 106  *) Get rid of write barriers for indirect stores.  We can do this by
 107     telling the GC to wbarrier-register an object once we do an ldloca
 108     or ldelema on it, and to unregister it once it's not used anymore
 109     (it can only travel downwards on the stack).  The problem with
 110     unregistering is that it needs to happen eventually no matter
 111     what, even if exceptions are thrown, the thread aborts, etc.
 112     Rodrigo suggested that we could do only the registering part and
 113     let the collector find out (pessimistically) when it's safe to
 114     unregister, namely when the stack pointer of the thread that
 115     registered the object is higher than it was when the registering
 116     happened.  This might make for a good first implementation to get
 117     some data on performance.
 118
 119  *) Some sort of blacklist support?  Blacklists is a concept from the
 120     Boehm GC: if during a conservative scan we find pointers to an
 121     area which we might use as heap, we mark that area as unusable, so
 122     pointer retention by random pinning pointers is reduced.
 123
 124  *) experiment with max small object size (very small right now - 2kb,
 125     because it's tied to the max freelist size)
 126
 127   *) add an option to mmap the whole heap in one chunk: it makes for many
 128      simplifications in the checks (put the nursery at the top and just use a single
 129      check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
 130      not flexible (too much of the address space may be used by default or we can't
 131      increase the heap as needed) and we'd need a race-free mechanism to return memory
 132      back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
 133      was written to, munmap is needed, but the following mmap may not find the same segment
 134      free...)
 135
 136  *) memzero the major fragments after restarting the world and optionally a smaller
 137     chunk at a time
 138
 139  *) investigate having fragment zeroing threads
 140
 141  *) separate locks for finalization and other minor stuff to reduce
 142     lock contention
 143
 144  *) try a different copying order to improve memory locality
 145
 146  *) a thread abort after a store but before the write barrier will
 147     prevent the write barrier from executing
 148
 149  *) specialized dynamically generated markers/copiers
 150
 151  *) Dynamically adjust TLAB size to the number of threads.  If we have
 152     too many threads that do allocation, we might need smaller TLABs,
 153     and we might get better performance with larger TLABs if we only
 154     have a handful of threads.  We could sum up the space left in all
 155     assigned TLABs and if that's more than some percentage of the
 156     nursery size, reduce the TLAB size.
 157
 158  *) Explore placing unreachable objects on unused nursery memory.
 159         Instead of memset'ng a region to zero, place an int[] covering it.
 160         A good place to start is add_nursery_frag. The tricky thing here is
 161         placing those objects atomically outside of a collection.
 162
 163  *) Allocation should use asymmetric Dekker synchronization:
 164         http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
 165         This should help weak consistency archs.
 166  */
 167 #include "config.h"
 168 #ifdef HAVE_SGEN_GC
 169
 170 #ifdef __MACH__
 171 #undef _XOPEN_SOURCE
 172 #define _XOPEN_SOURCE
 173 #define _DARWIN_C_SOURCE
 174 #endif
 175
 176 #ifdef HAVE_UNISTD_H
 177 #include <unistd.h>
 178 #endif
 179 #ifdef HAVE_PTHREAD_H
 180 #include <pthread.h>
 181 #endif
 182 #ifdef HAVE_PTHREAD_NP_H
 183 #include <pthread_np.h>
 184 #endif
 185 #ifdef HAVE_SEMAPHORE_H
 186 #include <semaphore.h>
 187 #endif
 188 #include <stdio.h>
 189 #include <string.h>
 190 #include <signal.h>
 191 #include <errno.h>
 192 #include <assert.h>
 193
 194 #include "metadata/sgen-gc.h"
 195 #include "metadata/metadata-internals.h"
 196 #include "metadata/class-internals.h"
 197 #include "metadata/gc-internal.h"
 198 #include "metadata/object-internals.h"
 199 #include "metadata/threads.h"
 200 #include "metadata/sgen-cardtable.h"
 201 #include "metadata/sgen-ssb.h"
 202 #include "metadata/sgen-protocol.h"
 203 #include "metadata/sgen-archdep.h"
 204 #include "metadata/sgen-bridge.h"
 205 #include "metadata/sgen-memory-governor.h"
 206 #include "metadata/sgen-hash-table.h"
 207 #include "metadata/mono-gc.h"
 208 #include "metadata/method-builder.h"
 209 #include "metadata/profiler-private.h"
 210 #include "metadata/monitor.h"
 211 #include "metadata/threadpool-internals.h"
 212 #include "metadata/mempool-internals.h"
 213 #include "metadata/marshal.h"
 214 #include "metadata/runtime.h"
 215 #include "metadata/sgen-cardtable.h"
 216 #include "metadata/sgen-pinning.h"
 217 #include "metadata/sgen-workers.h"
 218 #include "utils/mono-mmap.h"
 219 #include "utils/mono-time.h"
 220 #include "utils/mono-semaphore.h"
 221 #include "utils/mono-counters.h"
 222 #include "utils/mono-proclib.h"
 223 #include "utils/mono-memory-model.h"
 224 #include "utils/mono-logger-internal.h"
 225 #include "utils/dtrace.h"
 226
 227 #include <mono/utils/mono-logger-internal.h>
 228 #include <mono/utils/memcheck.h>
 229
 230 #if defined(__MACH__)
 231 #include "utils/mach-support.h"
 232 #endif
 233
 234 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
 235         a = i,
 236
 237 enum {
 238 #include "mono/cil/opcode.def"
 239         CEE_LAST
 240 };
 241
 242 #undef OPDEF
 243
 244 #undef pthread_create
 245 #undef pthread_join
 246 #undef pthread_detach
 247
 248 /*
 249  * ######################################################################
 250  * ########  Types and constants used by the GC.
 251  * ######################################################################
 252  */
 253
 254 /* 0 means not initialized, 1 is initialized, -1 means in progress */
 255 static int gc_initialized = 0;
 256 /* If set, check if we need to do something every X allocations */
 257 gboolean has_per_allocation_action;
 258 /* If set, do a heap check every X allocation */
 259 guint32 verify_before_allocs = 0;
 260 /* If set, do a minor collection before every X allocation */
 261 guint32 collect_before_allocs = 0;
 262 /* If set, do a whole heap check before each collection */
 263 static gboolean whole_heap_check_before_collection = FALSE;
 264 /* If set, do a heap consistency check before each minor collection */
 265 static gboolean consistency_check_at_minor_collection = FALSE;
 266 /* If set, check whether mark bits are consistent after major collections */
 267 static gboolean check_mark_bits_after_major_collection = FALSE;
 268 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
 269 static gboolean check_nursery_objects_pinned = FALSE;
 270 /* If set, do a few checks when the concurrent collector is used */
 271 static gboolean do_concurrent_checks = FALSE;
 272 /* If set, check that there are no references to the domain left at domain unload */
 273 static gboolean xdomain_checks = FALSE;
 274 /* If not null, dump the heap after each collection into this file */
 275 static FILE *heap_dump_file = NULL;
 276 /* If set, mark stacks conservatively, even if precise marking is possible */
 277 static gboolean conservative_stack_mark = FALSE;
 278 /* If set, do a plausibility check on the scan_starts before and after
 279    each collection */
 280 static gboolean do_scan_starts_check = FALSE;
 281 static gboolean nursery_collection_is_parallel = FALSE;
 282 static gboolean disable_minor_collections = FALSE;
 283 static gboolean disable_major_collections = FALSE;
 284 gboolean do_pin_stats = FALSE;
 285 static gboolean do_verify_nursery = FALSE;
 286 static gboolean do_dump_nursery_content = FALSE;
 287
 288 #ifdef HEAVY_STATISTICS
 289 long long stat_objects_alloced_degraded = 0;
 290 long long stat_bytes_alloced_degraded = 0;
 291
 292 long long stat_copy_object_called_nursery = 0;
 293 long long stat_objects_copied_nursery = 0;
 294 long long stat_copy_object_called_major = 0;
 295 long long stat_objects_copied_major = 0;
 296
 297 long long stat_scan_object_called_nursery = 0;
 298 long long stat_scan_object_called_major = 0;
 299
 300 long long stat_slots_allocated_in_vain;
 301
 302 long long stat_nursery_copy_object_failed_from_space = 0;
 303 long long stat_nursery_copy_object_failed_forwarded = 0;
 304 long long stat_nursery_copy_object_failed_pinned = 0;
 305 long long stat_nursery_copy_object_failed_to_space = 0;
 306
 307 static int stat_wbarrier_set_field = 0;
 308 static int stat_wbarrier_set_arrayref = 0;
 309 static int stat_wbarrier_arrayref_copy = 0;
 310 static int stat_wbarrier_generic_store = 0;
 311 static int stat_wbarrier_set_root = 0;
 312 static int stat_wbarrier_value_copy = 0;
 313 static int stat_wbarrier_object_copy = 0;
 314 #endif
 315
 316 int stat_minor_gcs = 0;
 317 int stat_major_gcs = 0;
 318
 319 static long long stat_pinned_objects = 0;
 320
 321 static long long time_minor_pre_collection_fragment_clear = 0;
 322 static long long time_minor_pinning = 0;
 323 static long long time_minor_scan_remsets = 0;
 324 static long long time_minor_scan_pinned = 0;
 325 static long long time_minor_scan_registered_roots = 0;
 326 static long long time_minor_scan_thread_data = 0;
 327 static long long time_minor_finish_gray_stack = 0;
 328 static long long time_minor_fragment_creation = 0;
 329
 330 static long long time_major_pre_collection_fragment_clear = 0;
 331 static long long time_major_pinning = 0;
 332 static long long time_major_scan_pinned = 0;
 333 static long long time_major_scan_registered_roots = 0;
 334 static long long time_major_scan_thread_data = 0;
 335 static long long time_major_scan_alloc_pinned = 0;
 336 static long long time_major_scan_finalized = 0;
 337 static long long time_major_scan_big_objects = 0;
 338 static long long time_major_finish_gray_stack = 0;
 339 static long long time_major_free_bigobjs = 0;
 340 static long long time_major_los_sweep = 0;
 341 static long long time_major_sweep = 0;
 342 static long long time_major_fragment_creation = 0;
 343
 344 int gc_debug_level = 0;
 345 FILE* gc_debug_file;
 346
 347 /*
 348 void
 349 mono_gc_flush_info (void)
 350 {
 351         fflush (gc_debug_file);
 352 }
 353 */
 354
 355 #define TV_DECLARE SGEN_TV_DECLARE
 356 #define TV_GETTIME SGEN_TV_GETTIME
 357 #define TV_ELAPSED SGEN_TV_ELAPSED
 358 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
 359
 360 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
 361
 362 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
 363
 364 #define object_is_forwarded     SGEN_OBJECT_IS_FORWARDED
 365 #define object_is_pinned        SGEN_OBJECT_IS_PINNED
 366 #define pin_object              SGEN_PIN_OBJECT
 367 #define unpin_object            SGEN_UNPIN_OBJECT
 368
 369 #define ptr_in_nursery sgen_ptr_in_nursery
 370
 371 #define LOAD_VTABLE     SGEN_LOAD_VTABLE
 372
 373 static const char*
 374 safe_name (void* obj)
 375 {
 376         MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
 377         return vt->klass->name;
 378 }
 379
 380 #define safe_object_get_size    sgen_safe_object_get_size
 381
 382 const char*
 383 sgen_safe_name (void* obj)
 384 {
 385         return safe_name (obj);
 386 }
 387
 388 /*
 389  * ######################################################################
 390  * ########  Global data.
 391  * ######################################################################
 392  */
 393 LOCK_DECLARE (gc_mutex);
 394
 395 static gboolean use_cardtable;
 396
 397 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
 398
 399 static mword pagesize = 4096;
 400 int degraded_mode = 0;
 401
 402 static mword bytes_pinned_from_failed_allocation = 0;
 403
 404 GCMemSection *nursery_section = NULL;
 405 static mword lowest_heap_address = ~(mword)0;
 406 static mword highest_heap_address = 0;
 407
 408 LOCK_DECLARE (sgen_interruption_mutex);
 409 static LOCK_DECLARE (pin_queue_mutex);
 410
 411 #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex)
 412 #define UNLOCK_PIN_QUEUE mono_mutex_unlock (&pin_queue_mutex)
 413
 414 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
 415 struct _FinalizeReadyEntry {
 416         FinalizeReadyEntry *next;
 417         void *object;
 418 };
 419
 420 typedef struct _EphemeronLinkNode EphemeronLinkNode;
 421
 422 struct _EphemeronLinkNode {
 423         EphemeronLinkNode *next;
 424         char *array;
 425 };
 426
 427 typedef struct {
 428        void *key;
 429        void *value;
 430 } Ephemeron;
 431
 432 int current_collection_generation = -1;
 433 volatile gboolean concurrent_collection_in_progress = FALSE;
 434
 435 /* objects that are ready to be finalized */
 436 static FinalizeReadyEntry *fin_ready_list = NULL;
 437 static FinalizeReadyEntry *critical_fin_list = NULL;
 438
 439 static EphemeronLinkNode *ephemeron_list;
 440
 441 /* registered roots: the key to the hash is the root start address */
 442 /*
 443  * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
 444  */
 445 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
 446         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 447         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 448         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
 449 };
 450 static mword roots_size = 0; /* amount of memory in the root set */
 451
 452 #define GC_ROOT_NUM 32
 453 typedef struct {
 454         int count;              /* must be the first field */
 455         void *objects [GC_ROOT_NUM];
 456         int root_types [GC_ROOT_NUM];
 457         uintptr_t extra_info [GC_ROOT_NUM];
 458 } GCRootReport;
 459
 460 static void
 461 notify_gc_roots (GCRootReport *report)
 462 {
 463         if (!report->count)
 464                 return;
 465         mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
 466         report->count = 0;
 467 }
 468
 469 static void
 470 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
 471 {
 472         if (report->count == GC_ROOT_NUM)
 473                 notify_gc_roots (report);
 474         report->objects [report->count] = object;
 475         report->root_types [report->count] = rtype;
 476         report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
 477 }
 478
 479 MonoNativeTlsKey thread_info_key;
 480
 481 #ifdef HAVE_KW_THREAD
 482 __thread SgenThreadInfo *sgen_thread_info;
 483 __thread gpointer *store_remset_buffer;
 484 __thread long store_remset_buffer_index;
 485 __thread char *stack_end;
 486 __thread long *store_remset_buffer_index_addr;
 487 #endif
 488
 489 /* The size of a TLAB */
 490 /* The bigger the value, the less often we have to go to the slow path to allocate a new
 491  * one, but the more space is wasted by threads not allocating much memory.
 492  * FIXME: Tune this.
 493  * FIXME: Make this self-tuning for each thread.
 494  */
 495 guint32 tlab_size = (1024 * 4);
 496
 497 #define MAX_SMALL_OBJ_SIZE      SGEN_MAX_SMALL_OBJ_SIZE
 498
 499 /* Functions supplied by the runtime to be called by the GC */
 500 static MonoGCCallbacks gc_callbacks;
 501
 502 #define ALLOC_ALIGN             SGEN_ALLOC_ALIGN
 503 #define ALLOC_ALIGN_BITS        SGEN_ALLOC_ALIGN_BITS
 504
 505 #define ALIGN_UP                SGEN_ALIGN_UP
 506
 507 #define MOVED_OBJECTS_NUM 64
 508 static void *moved_objects [MOVED_OBJECTS_NUM];
 509 static int moved_objects_idx = 0;
 510
 511 /* Vtable of the objects used to fill out nursery fragments before a collection */
 512 static MonoVTable *array_fill_vtable;
 513
 514 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
 515 MonoNativeThreadId main_gc_thread = NULL;
 516 #endif
 517
 518 /*Object was pinned during the current collection*/
 519 static mword objects_pinned;
 520
 521 /*
 522  * ######################################################################
 523  * ########  Macros and function declarations.
 524  * ######################################################################
 525  */
 526
 527 inline static void*
 528 align_pointer (void *ptr)
 529 {
 530         mword p = (mword)ptr;
 531         p += sizeof (gpointer) - 1;
 532         p &= ~ (sizeof (gpointer) - 1);
 533         return (void*)p;
 534 }
 535
 536 typedef SgenGrayQueue GrayQueue;
 537
 538 /* forward declarations */
 539 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
 540 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
 541 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
 542 static void report_finalizer_roots (void);
 543 static void report_registered_roots (void);
 544
 545 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
 546 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx);
 547 static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue);
 548
 549 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
 550
 551
 552 static void init_stats (void);
 553
 554 static int mark_ephemerons_in_range (ScanCopyContext ctx);
 555 static void clear_unreachable_ephemerons (ScanCopyContext ctx);
 556 static void null_ephemerons_for_domain (MonoDomain *domain);
 557
 558 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
 559
 560 SgenObjectOperations current_object_ops;
 561 SgenMajorCollector major_collector;
 562 SgenMinorCollector sgen_minor_collector;
 563 static GrayQueue gray_queue;
 564
 565 static SgenRemeberedSet remset;
 566
 567 /* The gray queue to use from the main collection thread. */
 568 #define WORKERS_DISTRIBUTE_GRAY_QUEUE   (&gray_queue)
 569
 570 /*
 571  * The gray queue a worker job must use.  If we're not parallel or
 572  * concurrent, we use the main gray queue.
 573  */
 574 static SgenGrayQueue*
 575 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
 576 {
 577         return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
 578 }
 579
 580 static void
 581 gray_queue_redirect (SgenGrayQueue *queue)
 582 {
 583         gboolean wake = FALSE;
 584
 585
 586         for (;;) {
 587                 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
 588                 if (!section)
 589                         break;
 590                 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
 591                 wake = TRUE;
 592         }
 593
 594         if (wake) {
 595                 g_assert (concurrent_collection_in_progress ||
 596                                 (current_collection_generation == GENERATION_OLD && major_collector.is_parallel));
 597                 if (sgen_workers_have_started ()) {
 598                         sgen_workers_wake_up_all ();
 599                 } else {
 600                         if (concurrent_collection_in_progress)
 601                                 g_assert (current_collection_generation == -1);
 602                 }
 603         }
 604 }
 605
 606 static gboolean
 607 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
 608 {
 609         MonoObject *o = (MonoObject*)(obj);
 610         MonoObject *ref = (MonoObject*)*(ptr);
 611         int offset = (char*)(ptr) - (char*)o;
 612
 613         if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
 614                 return TRUE;
 615         if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
 616                 return TRUE;
 617         if (mono_class_has_parent_fast (o->vtable->klass, mono_defaults.real_proxy_class) &&
 618                         offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
 619                 return TRUE;
 620         /* Thread.cached_culture_info */
 621         if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
 622                         !strcmp (ref->vtable->klass->name, "CultureInfo") &&
 623                         !strcmp(o->vtable->klass->name_space, "System") &&
 624                         !strcmp(o->vtable->klass->name, "Object[]"))
 625                 return TRUE;
 626         /*
 627          *  at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
 628          * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
 629          * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
 630          * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
 631          * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
 632          * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
 633          * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
 634          * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
 635          * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
 636          */
 637         if (!strcmp (ref->vtable->klass->name_space, "System") &&
 638                         !strcmp (ref->vtable->klass->name, "Byte[]") &&
 639                         !strcmp (o->vtable->klass->name_space, "System.IO") &&
 640                         !strcmp (o->vtable->klass->name, "MemoryStream"))
 641                 return TRUE;
 642         /* append_job() in threadpool.c */
 643         if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
 644                         !strcmp (ref->vtable->klass->name, "AsyncResult") &&
 645                         !strcmp (o->vtable->klass->name_space, "System") &&
 646                         !strcmp (o->vtable->klass->name, "Object[]") &&
 647                         mono_thread_pool_is_queue_array ((MonoArray*) o))
 648                 return TRUE;
 649         return FALSE;
 650 }
 651
 652 static void
 653 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
 654 {
 655         MonoObject *o = (MonoObject*)(obj);
 656         MonoObject *ref = (MonoObject*)*(ptr);
 657         int offset = (char*)(ptr) - (char*)o;
 658         MonoClass *class;
 659         MonoClassField *field;
 660         char *str;
 661
 662         if (!ref || ref->vtable->domain == domain)
 663                 return;
 664         if (is_xdomain_ref_allowed (ptr, obj, domain))
 665                 return;
 666
 667         field = NULL;
 668         for (class = o->vtable->klass; class; class = class->parent) {
 669                 int i;
 670
 671                 for (i = 0; i < class->field.count; ++i) {
 672                         if (class->fields[i].offset == offset) {
 673                                 field = &class->fields[i];
 674                                 break;
 675                         }
 676                 }
 677                 if (field)
 678                         break;
 679         }
 680
 681         if (ref->vtable->klass == mono_defaults.string_class)
 682                 str = mono_string_to_utf8 ((MonoString*)ref);
 683         else
 684                 str = NULL;
 685         g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s)  -  pointed to by:\n",
 686                         o, o->vtable->klass->name_space, o->vtable->klass->name,
 687                         offset, field ? field->name : "",
 688                         ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
 689         mono_gc_scan_for_specific_ref (o, TRUE);
 690         if (str)
 691                 g_free (str);
 692 }
 693
 694 #undef HANDLE_PTR
 695 #define HANDLE_PTR(ptr,obj)     check_reference_for_xdomain ((ptr), (obj), domain)
 696
 697 static void
 698 scan_object_for_xdomain_refs (char *start, mword size, void *data)
 699 {
 700         MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
 701
 702         #include "sgen-scan-object.h"
 703 }
 704
 705 static gboolean scan_object_for_specific_ref_precise = TRUE;
 706
 707 #undef HANDLE_PTR
 708 #define HANDLE_PTR(ptr,obj) do {                \
 709         if ((MonoObject*)*(ptr) == key) {       \
 710         g_print ("found ref to %p in object %p (%s) at offset %td\n",   \
 711                         key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
 712         }                                                               \
 713         } while (0)
 714
 715 static void
 716 scan_object_for_specific_ref (char *start, MonoObject *key)
 717 {
 718         char *forwarded;
 719
 720         if ((forwarded = SGEN_OBJECT_IS_FORWARDED (start)))
 721                 start = forwarded;
 722
 723         if (scan_object_for_specific_ref_precise) {
 724                 #include "sgen-scan-object.h"
 725         } else {
 726                 mword *words = (mword*)start;
 727                 size_t size = safe_object_get_size ((MonoObject*)start);
 728                 int i;
 729                 for (i = 0; i < size / sizeof (mword); ++i) {
 730                         if (words [i] == (mword)key) {
 731                                 g_print ("found possible ref to %p in object %p (%s) at offset %td\n",
 732                                                 key, start, safe_name (start), i * sizeof (mword));
 733                         }
 734                 }
 735         }
 736 }
 737
 738 void
 739 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
 740 {
 741         while (start < end) {
 742                 size_t size;
 743                 char *obj;
 744
 745                 if (!*(void**)start) {
 746                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
 747                         continue;
 748                 }
 749
 750                 if (allow_flags) {
 751                         if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
 752                                 obj = start;
 753                 } else {
 754                         obj = start;
 755                 }
 756
 757                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
 758
 759                 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable)
 760                         callback (obj, size, data);
 761
 762                 start += size;
 763         }
 764 }
 765
 766 static void
 767 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
 768 {
 769         scan_object_for_specific_ref (obj, key);
 770 }
 771
 772 static void
 773 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
 774 {
 775         if (key != obj)
 776                 return;
 777         g_print ("found ref to %p in root record %p\n", key, root);
 778 }
 779
 780 static MonoObject *check_key = NULL;
 781 static RootRecord *check_root = NULL;
 782
 783 static void
 784 check_root_obj_specific_ref_from_marker (void **obj)
 785 {
 786         check_root_obj_specific_ref (check_root, check_key, *obj);
 787 }
 788
 789 static void
 790 scan_roots_for_specific_ref (MonoObject *key, int root_type)
 791 {
 792         void **start_root;
 793         RootRecord *root;
 794         check_key = key;
 795
 796         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 797                 mword desc = root->root_desc;
 798
 799                 check_root = root;
 800
 801                 switch (desc & ROOT_DESC_TYPE_MASK) {
 802                 case ROOT_DESC_BITMAP:
 803                         desc >>= ROOT_DESC_TYPE_SHIFT;
 804                         while (desc) {
 805                                 if (desc & 1)
 806                                         check_root_obj_specific_ref (root, key, *start_root);
 807                                 desc >>= 1;
 808                                 start_root++;
 809                         }
 810                         return;
 811                 case ROOT_DESC_COMPLEX: {
 812                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 813                         int bwords = (*bitmap_data) - 1;
 814                         void **start_run = start_root;
 815                         bitmap_data++;
 816                         while (bwords-- > 0) {
 817                                 gsize bmap = *bitmap_data++;
 818                                 void **objptr = start_run;
 819                                 while (bmap) {
 820                                         if (bmap & 1)
 821                                                 check_root_obj_specific_ref (root, key, *objptr);
 822                                         bmap >>= 1;
 823                                         ++objptr;
 824                                 }
 825                                 start_run += GC_BITS_PER_WORD;
 826                         }
 827                         break;
 828                 }
 829                 case ROOT_DESC_USER: {
 830                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 831                         marker (start_root, check_root_obj_specific_ref_from_marker);
 832                         break;
 833                 }
 834                 case ROOT_DESC_RUN_LEN:
 835                         g_assert_not_reached ();
 836                 default:
 837                         g_assert_not_reached ();
 838                 }
 839         } SGEN_HASH_TABLE_FOREACH_END;
 840
 841         check_key = NULL;
 842         check_root = NULL;
 843 }
 844
 845 void
 846 mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise)
 847 {
 848         void **ptr;
 849         RootRecord *root;
 850
 851         scan_object_for_specific_ref_precise = precise;
 852
 853         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 854                         (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key, TRUE);
 855
 856         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 857
 858         sgen_los_iterate_objects ((IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 859
 860         scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
 861         scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
 862
 863         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], ptr, root) {
 864                 while (ptr < (void**)root->end_root) {
 865                         check_root_obj_specific_ref (root, *ptr, key);
 866                         ++ptr;
 867                 }
 868         } SGEN_HASH_TABLE_FOREACH_END;
 869 }
 870
 871 static gboolean
 872 need_remove_object_for_domain (char *start, MonoDomain *domain)
 873 {
 874         if (mono_object_domain (start) == domain) {
 875                 SGEN_LOG (4, "Need to cleanup object %p", start);
 876                 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
 877                 return TRUE;
 878         }
 879         return FALSE;
 880 }
 881
 882 static void
 883 process_object_for_domain_clearing (char *start, MonoDomain *domain)
 884 {
 885         GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
 886         if (vt->klass == mono_defaults.internal_thread_class)
 887                 g_assert (mono_object_domain (start) == mono_get_root_domain ());
 888         /* The object could be a proxy for an object in the domain
 889            we're deleting. */
 890         if (mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
 891                 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
 892
 893                 /* The server could already have been zeroed out, so
 894                    we need to check for that, too. */
 895                 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
 896                         SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
 897                         ((MonoRealProxy*)start)->unwrapped_server = NULL;
 898                 }
 899         }
 900 }
 901
 902 static MonoDomain *check_domain = NULL;
 903
 904 static void
 905 check_obj_not_in_domain (void **o)
 906 {
 907         g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
 908 }
 909
 910 static void
 911 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
 912 {
 913         void **start_root;
 914         RootRecord *root;
 915         check_domain = domain;
 916         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 917                 mword desc = root->root_desc;
 918
 919                 /* The MonoDomain struct is allowed to hold
 920                    references to objects in its own domain. */
 921                 if (start_root == (void**)domain)
 922                         continue;
 923
 924                 switch (desc & ROOT_DESC_TYPE_MASK) {
 925                 case ROOT_DESC_BITMAP:
 926                         desc >>= ROOT_DESC_TYPE_SHIFT;
 927                         while (desc) {
 928                                 if ((desc & 1) && *start_root)
 929                                         check_obj_not_in_domain (*start_root);
 930                                 desc >>= 1;
 931                                 start_root++;
 932                         }
 933                         break;
 934                 case ROOT_DESC_COMPLEX: {
 935                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 936                         int bwords = (*bitmap_data) - 1;
 937                         void **start_run = start_root;
 938                         bitmap_data++;
 939                         while (bwords-- > 0) {
 940                                 gsize bmap = *bitmap_data++;
 941                                 void **objptr = start_run;
 942                                 while (bmap) {
 943                                         if ((bmap & 1) && *objptr)
 944                                                 check_obj_not_in_domain (*objptr);
 945                                         bmap >>= 1;
 946                                         ++objptr;
 947                                 }
 948                                 start_run += GC_BITS_PER_WORD;
 949                         }
 950                         break;
 951                 }
 952                 case ROOT_DESC_USER: {
 953                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 954                         marker (start_root, check_obj_not_in_domain);
 955                         break;
 956                 }
 957                 case ROOT_DESC_RUN_LEN:
 958                         g_assert_not_reached ();
 959                 default:
 960                         g_assert_not_reached ();
 961                 }
 962         } SGEN_HASH_TABLE_FOREACH_END;
 963
 964         check_domain = NULL;
 965 }
 966
 967 static void
 968 check_for_xdomain_refs (void)
 969 {
 970         LOSObject *bigobj;
 971
 972         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 973                         (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL, FALSE);
 974
 975         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
 976
 977         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
 978                 scan_object_for_xdomain_refs (bigobj->data, sgen_los_object_size (bigobj), NULL);
 979 }
 980
 981 static gboolean
 982 clear_domain_process_object (char *obj, MonoDomain *domain)
 983 {
 984         gboolean remove;
 985
 986         process_object_for_domain_clearing (obj, domain);
 987         remove = need_remove_object_for_domain (obj, domain);
 988
 989         if (remove && ((MonoObject*)obj)->synchronisation) {
 990                 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
 991                 if (dislink)
 992                         sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
 993         }
 994
 995         return remove;
 996 }
 997
 998 static void
 999 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1000 {
1001         if (clear_domain_process_object (obj, domain))
1002                 memset (obj, 0, size);
1003 }
1004
1005 static void
1006 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1007 {
1008         clear_domain_process_object (obj, domain);
1009 }
1010
1011 static void
1012 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1013 {
1014         if (need_remove_object_for_domain (obj, domain))
1015                 major_collector.free_non_pinned_object (obj, size);
1016 }
1017
1018 static void
1019 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1020 {
1021         if (need_remove_object_for_domain (obj, domain))
1022                 major_collector.free_pinned_object (obj, size);
1023 }
1024
1025 /*
1026  * When appdomains are unloaded we can easily remove objects that have finalizers,
1027  * but all the others could still be present in random places on the heap.
1028  * We need a sweep to get rid of them even though it's going to be costly
1029  * with big heaps.
1030  * The reason we need to remove them is because we access the vtable and class
1031  * structures to know the object size and the reference bitmap: once the domain is
1032  * unloaded the point to random memory.
1033  */
1034 void
1035 mono_gc_clear_domain (MonoDomain * domain)
1036 {
1037         LOSObject *bigobj, *prev;
1038         int i;
1039
1040         LOCK_GC;
1041
1042         if (concurrent_collection_in_progress)
1043                 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
1044         g_assert (!concurrent_collection_in_progress);
1045
1046         sgen_process_fin_stage_entries ();
1047         sgen_process_dislink_stage_entries ();
1048
1049         sgen_clear_nursery_fragments ();
1050
1051         if (xdomain_checks && domain != mono_get_root_domain ()) {
1052                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1053                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1054                 check_for_xdomain_refs ();
1055         }
1056
1057         /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1058         to memory returned to the OS.*/
1059         null_ephemerons_for_domain (domain);
1060
1061         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1062                 sgen_null_links_for_domain (domain, i);
1063
1064         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1065                 sgen_remove_finalizers_for_domain (domain, i);
1066
1067         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1068                         (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
1069
1070         /* We need two passes over major and large objects because
1071            freeing such objects might give their memory back to the OS
1072            (in the case of large objects) or obliterate its vtable
1073            (pinned objects with major-copying or pinned and non-pinned
1074            objects with major-mark&sweep), but we might need to
1075            dereference a pointer from an object to another object if
1076            the first object is a proxy. */
1077         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1078         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1079                 clear_domain_process_object (bigobj->data, domain);
1080
1081         prev = NULL;
1082         for (bigobj = los_object_list; bigobj;) {
1083                 if (need_remove_object_for_domain (bigobj->data, domain)) {
1084                         LOSObject *to_free = bigobj;
1085                         if (prev)
1086                                 prev->next = bigobj->next;
1087                         else
1088                                 los_object_list = bigobj->next;
1089                         bigobj = bigobj->next;
1090                         SGEN_LOG (4, "Freeing large object %p", bigobj->data);
1091                         sgen_los_free_object (to_free);
1092                         continue;
1093                 }
1094                 prev = bigobj;
1095                 bigobj = bigobj->next;
1096         }
1097         major_collector.iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1098         major_collector.iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1099
1100         if (G_UNLIKELY (do_pin_stats)) {
1101                 if (domain == mono_get_root_domain ())
1102                         sgen_pin_stats_print_class_stats ();
1103         }
1104
1105         UNLOCK_GC;
1106 }
1107
1108 /*
1109  * sgen_add_to_global_remset:
1110  *
1111  *   The global remset contains locations which point into newspace after
1112  * a minor collection. This can happen if the objects they point to are pinned.
1113  *
1114  * LOCKING: If called from a parallel collector, the global remset
1115  * lock must be held.  For serial collectors that is not necessary.
1116  */
1117 void
1118 sgen_add_to_global_remset (gpointer ptr, gpointer obj)
1119 {
1120         SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
1121
1122         if (!major_collector.is_concurrent) {
1123                 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
1124         } else {
1125                 if (current_collection_generation == -1)
1126                         SGEN_ASSERT (5, sgen_concurrent_collection_in_progress (), "Global remsets outside of collection pauses can only be added by the concurrent collector");
1127         }
1128
1129         if (!object_is_pinned (obj))
1130                 SGEN_ASSERT (5, sgen_minor_collector.is_split || sgen_concurrent_collection_in_progress (), "Non-pinned objects can only remain in nursery if it is a split nursery");
1131         else if (sgen_cement_lookup_or_register (obj))
1132                 return;
1133
1134         remset.record_pointer (ptr);
1135
1136         if (G_UNLIKELY (do_pin_stats))
1137                 sgen_pin_stats_register_global_remset (obj);
1138
1139         SGEN_LOG (8, "Adding global remset for %p", ptr);
1140         binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
1141
1142         HEAVY_STAT (++stat_global_remsets_added);
1143
1144 #ifdef ENABLE_DTRACE
1145         if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
1146                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
1147                 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
1148                                 vt->klass->name_space, vt->klass->name);
1149         }
1150 #endif
1151 }
1152
1153 /*
1154  * sgen_drain_gray_stack:
1155  *
1156  *   Scan objects in the gray stack until the stack is empty. This should be called
1157  * frequently after each object is copied, to achieve better locality and cache
1158  * usage.
1159  */
1160 gboolean
1161 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
1162 {
1163         char *obj;
1164         ScanObjectFunc scan_func = ctx.scan_func;
1165         GrayQueue *queue = ctx.queue;
1166
1167         if (max_objs == -1) {
1168                 for (;;) {
1169                         GRAY_OBJECT_DEQUEUE (queue, obj);
1170                         if (!obj)
1171                                 return TRUE;
1172                         SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1173                         scan_func (obj, queue);
1174                 }
1175         } else {
1176                 int i;
1177
1178                 do {
1179                         for (i = 0; i != max_objs; ++i) {
1180                                 GRAY_OBJECT_DEQUEUE (queue, obj);
1181                                 if (!obj)
1182                                         return TRUE;
1183                                 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1184                                 scan_func (obj, queue);
1185                         }
1186                 } while (max_objs < 0);
1187                 return FALSE;
1188         }
1189 }
1190
1191 /*
1192  * Addresses from start to end are already sorted. This function finds
1193  * the object header for each address and pins the object. The
1194  * addresses must be inside the passed section.  The (start of the)
1195  * address array is overwritten with the addresses of the actually
1196  * pinned objects.  Return the number of pinned objects.
1197  */
1198 static int
1199 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx)
1200 {
1201         void *last = NULL;
1202         int count = 0;
1203         void *search_start;
1204         void *last_obj = NULL;
1205         size_t last_obj_size = 0;
1206         void *addr;
1207         int idx;
1208         void **definitely_pinned = start;
1209         ScanObjectFunc scan_func = ctx.scan_func;
1210         SgenGrayQueue *queue = ctx.queue;
1211
1212         sgen_nursery_allocator_prepare_for_pinning ();
1213
1214         while (start < end) {
1215                 addr = *start;
1216                 /* the range check should be reduntant */
1217                 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1218                         SGEN_LOG (5, "Considering pinning addr %p", addr);
1219                         /* multiple pointers to the same object */
1220                         if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1221                                 start++;
1222                                 continue;
1223                         }
1224                         idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1225                         g_assert (idx < section->num_scan_start);
1226                         search_start = (void*)section->scan_starts [idx];
1227                         if (!search_start || search_start > addr) {
1228                                 while (idx) {
1229                                         --idx;
1230                                         search_start = section->scan_starts [idx];
1231                                         if (search_start && search_start <= addr)
1232                                                 break;
1233                                 }
1234                                 if (!search_start || search_start > addr)
1235                                         search_start = start_nursery;
1236                         }
1237                         if (search_start < last_obj)
1238                                 search_start = (char*)last_obj + last_obj_size;
1239                         /* now addr should be in an object a short distance from search_start
1240                          * Note that search_start must point to zeroed mem or point to an object.
1241                          */
1242
1243                         do {
1244                                 if (!*(void**)search_start) {
1245                                         /* Consistency check */
1246                                         /*
1247                                         for (frag = nursery_fragments; frag; frag = frag->next) {
1248                                                 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1249                                                         g_assert_not_reached ();
1250                                         }
1251                                         */
1252
1253                                         search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
1254                                         continue;
1255                                 }
1256                                 last_obj = search_start;
1257                                 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1258
1259                                 if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) {
1260                                         /* Marks the beginning of a nursery fragment, skip */
1261                                 } else {
1262                                         SGEN_LOG (8, "Pinned try match %p (%s), size %zd", last_obj, safe_name (last_obj), last_obj_size);
1263                                         if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1264                                                 if (scan_func) {
1265                                                         scan_func (search_start, queue);
1266                                                 } else {
1267                                                         SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1268                                                                         search_start, *(void**)search_start, safe_name (search_start), count);
1269                                                         binary_protocol_pin (search_start,
1270                                                                         (gpointer)LOAD_VTABLE (search_start),
1271                                                                         safe_object_get_size (search_start));
1272
1273 #ifdef ENABLE_DTRACE
1274                                                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1275                                                                 int gen = sgen_ptr_in_nursery (search_start) ? GENERATION_NURSERY : GENERATION_OLD;
1276                                                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (search_start);
1277                                                                 MONO_GC_OBJ_PINNED ((mword)search_start,
1278                                                                                 sgen_safe_object_get_size (search_start),
1279                                                                                 vt->klass->name_space, vt->klass->name, gen);
1280                                                         }
1281 #endif
1282
1283                                                         pin_object (search_start);
1284                                                         GRAY_OBJECT_ENQUEUE (queue, search_start);
1285                                                         if (G_UNLIKELY (do_pin_stats))
1286                                                                 sgen_pin_stats_register_object (search_start, last_obj_size);
1287                                                         definitely_pinned [count] = search_start;
1288                                                         count++;
1289                                                 }
1290                                                 break;
1291                                         }
1292                                 }
1293                                 /* skip to the next object */
1294                                 search_start = (void*)((char*)search_start + last_obj_size);
1295                         } while (search_start <= addr);
1296                         /* we either pinned the correct object or we ignored the addr because
1297                          * it points to unused zeroed memory.
1298                          */
1299                         last = addr;
1300                 }
1301                 start++;
1302         }
1303         //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1304         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1305                 GCRootReport report;
1306                 report.count = 0;
1307                 for (idx = 0; idx < count; ++idx)
1308                         add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1309                 notify_gc_roots (&report);
1310         }
1311         stat_pinned_objects += count;
1312         return count;
1313 }
1314
1315 void
1316 sgen_pin_objects_in_section (GCMemSection *section, ScanCopyContext ctx)
1317 {
1318         int num_entries = section->pin_queue_num_entries;
1319         if (num_entries) {
1320                 void **start = section->pin_queue_start;
1321                 int reduced_to;
1322                 reduced_to = pin_objects_from_addresses (section, start, start + num_entries,
1323                                 section->data, section->next_data, ctx);
1324                 section->pin_queue_num_entries = reduced_to;
1325                 if (!reduced_to)
1326                         section->pin_queue_start = NULL;
1327         }
1328 }
1329
1330
1331 void
1332 sgen_pin_object (void *object, GrayQueue *queue)
1333 {
1334         g_assert (!concurrent_collection_in_progress);
1335
1336         if (sgen_collection_is_parallel ()) {
1337                 LOCK_PIN_QUEUE;
1338                 /*object arrives pinned*/
1339                 sgen_pin_stage_ptr (object);
1340                 ++objects_pinned ;
1341                 UNLOCK_PIN_QUEUE;
1342         } else {
1343                 SGEN_PIN_OBJECT (object);
1344                 sgen_pin_stage_ptr (object);
1345                 ++objects_pinned;
1346                 if (G_UNLIKELY (do_pin_stats))
1347                         sgen_pin_stats_register_object (object, safe_object_get_size (object));
1348         }
1349         GRAY_OBJECT_ENQUEUE (queue, object);
1350         binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1351
1352 #ifdef ENABLE_DTRACE
1353         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1354                 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1355                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1356                 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1357         }
1358 #endif
1359 }
1360
1361 void
1362 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1363 {
1364         for (;;) {
1365                 mword vtable_word;
1366                 gboolean major_pinned = FALSE;
1367
1368                 if (sgen_ptr_in_nursery (obj)) {
1369                         if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
1370                                 sgen_pin_object (obj, queue);
1371                                 break;
1372                         }
1373                 } else {
1374                         major_collector.pin_major_object (obj, queue);
1375                         major_pinned = TRUE;
1376                 }
1377
1378                 vtable_word = *(mword*)obj;
1379                 /*someone else forwarded it, update the pointer and bail out*/
1380                 if (vtable_word & SGEN_FORWARDED_BIT) {
1381                         *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
1382                         break;
1383                 }
1384
1385                 /*someone pinned it, nothing to do.*/
1386                 if (vtable_word & SGEN_PINNED_BIT || major_pinned)
1387                         break;
1388         }
1389 }
1390
1391 /* Sort the addresses in array in increasing order.
1392  * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1393  */
1394 void
1395 sgen_sort_addresses (void **array, int size)
1396 {
1397         int i;
1398         void *tmp;
1399
1400         for (i = 1; i < size; ++i) {
1401                 int child = i;
1402                 while (child > 0) {
1403                         int parent = (child - 1) / 2;
1404
1405                         if (array [parent] >= array [child])
1406                                 break;
1407
1408                         tmp = array [parent];
1409                         array [parent] = array [child];
1410                         array [child] = tmp;
1411
1412                         child = parent;
1413                 }
1414         }
1415
1416         for (i = size - 1; i > 0; --i) {
1417                 int end, root;
1418                 tmp = array [i];
1419                 array [i] = array [0];
1420                 array [0] = tmp;
1421
1422                 end = i - 1;
1423                 root = 0;
1424
1425                 while (root * 2 + 1 <= end) {
1426                         int child = root * 2 + 1;
1427
1428                         if (child < end && array [child] < array [child + 1])
1429                                 ++child;
1430                         if (array [root] >= array [child])
1431                                 break;
1432
1433                         tmp = array [root];
1434                         array [root] = array [child];
1435                         array [child] = tmp;
1436
1437                         root = child;
1438                 }
1439         }
1440 }
1441
1442 /*
1443  * Scan the memory between start and end and queue values which could be pointers
1444  * to the area between start_nursery and end_nursery for later consideration.
1445  * Typically used for thread stacks.
1446  */
1447 static void
1448 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1449 {
1450         int count = 0;
1451
1452 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1453         VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1454 #endif
1455
1456         while (start < end) {
1457                 if (*start >= start_nursery && *start < end_nursery) {
1458                         /*
1459                          * *start can point to the middle of an object
1460                          * note: should we handle pointing at the end of an object?
1461                          * pinning in C# code disallows pointing at the end of an object
1462                          * but there is some small chance that an optimizing C compiler
1463                          * may keep the only reference to an object by pointing
1464                          * at the end of it. We ignore this small chance for now.
1465                          * Pointers to the end of an object are indistinguishable
1466                          * from pointers to the start of the next object in memory
1467                          * so if we allow that we'd need to pin two objects...
1468                          * We queue the pointer in an array, the
1469                          * array will then be sorted and uniqued. This way
1470                          * we can coalesce several pinning pointers and it should
1471                          * be faster since we'd do a memory scan with increasing
1472                          * addresses. Note: we can align the address to the allocation
1473                          * alignment, so the unique process is more effective.
1474                          */
1475                         mword addr = (mword)*start;
1476                         addr &= ~(ALLOC_ALIGN - 1);
1477                         if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1478                                 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1479                                 sgen_pin_stage_ptr ((void*)addr);
1480                                 count++;
1481                         }
1482                         if (G_UNLIKELY (do_pin_stats)) {
1483                                 if (ptr_in_nursery ((void*)addr))
1484                                         sgen_pin_stats_register_address ((char*)addr, pin_type);
1485                         }
1486                 }
1487                 start++;
1488         }
1489         if (count)
1490                 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1491 }
1492
1493 /*
1494  * The first thing we do in a collection is to identify pinned objects.
1495  * This function considers all the areas of memory that need to be
1496  * conservatively scanned.
1497  */
1498 static void
1499 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1500 {
1501         void **start_root;
1502         RootRecord *root;
1503         SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1504         /* objects pinned from the API are inside these roots */
1505         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1506                 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1507                 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1508         } SGEN_HASH_TABLE_FOREACH_END;
1509         /* now deal with the thread stacks
1510          * in the future we should be able to conservatively scan only:
1511          * *) the cpu registers
1512          * *) the unmanaged stack frames
1513          * *) the _last_ managed stack frame
1514          * *) pointers slots in managed frames
1515          */
1516         scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1517 }
1518
1519 static void
1520 unpin_objects_from_queue (SgenGrayQueue *queue)
1521 {
1522         for (;;) {
1523                 char *addr;
1524                 GRAY_OBJECT_DEQUEUE (queue, addr);
1525                 if (!addr)
1526                         break;
1527                 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1528                 SGEN_UNPIN_OBJECT (addr);
1529         }
1530 }
1531
1532 typedef struct {
1533         CopyOrMarkObjectFunc func;
1534         GrayQueue *queue;
1535 } UserCopyOrMarkData;
1536
1537 static MonoNativeTlsKey user_copy_or_mark_key;
1538
1539 static void
1540 init_user_copy_or_mark_key (void)
1541 {
1542         mono_native_tls_alloc (&user_copy_or_mark_key, NULL);
1543 }
1544
1545 static void
1546 set_user_copy_or_mark_data (UserCopyOrMarkData *data)
1547 {
1548         mono_native_tls_set_value (user_copy_or_mark_key, data);
1549 }
1550
1551 static void
1552 single_arg_user_copy_or_mark (void **obj)
1553 {
1554         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
1555
1556         data->func (obj, data->queue);
1557 }
1558
1559 /*
1560  * The memory area from start_root to end_root contains pointers to objects.
1561  * Their position is precisely described by @desc (this means that the pointer
1562  * can be either NULL or the pointer to the start of an object).
1563  * This functions copies them to to_space updates them.
1564  *
1565  * This function is not thread-safe!
1566  */
1567 static void
1568 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1569 {
1570         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1571         SgenGrayQueue *queue = ctx.queue;
1572
1573         switch (desc & ROOT_DESC_TYPE_MASK) {
1574         case ROOT_DESC_BITMAP:
1575                 desc >>= ROOT_DESC_TYPE_SHIFT;
1576                 while (desc) {
1577                         if ((desc & 1) && *start_root) {
1578                                 copy_func (start_root, queue);
1579                                 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1580                                 sgen_drain_gray_stack (-1, ctx);
1581                         }
1582                         desc >>= 1;
1583                         start_root++;
1584                 }
1585                 return;
1586         case ROOT_DESC_COMPLEX: {
1587                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1588                 int bwords = (*bitmap_data) - 1;
1589                 void **start_run = start_root;
1590                 bitmap_data++;
1591                 while (bwords-- > 0) {
1592                         gsize bmap = *bitmap_data++;
1593                         void **objptr = start_run;
1594                         while (bmap) {
1595                                 if ((bmap & 1) && *objptr) {
1596                                         copy_func (objptr, queue);
1597                                         SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1598                                         sgen_drain_gray_stack (-1, ctx);
1599                                 }
1600                                 bmap >>= 1;
1601                                 ++objptr;
1602                         }
1603                         start_run += GC_BITS_PER_WORD;
1604                 }
1605                 break;
1606         }
1607         case ROOT_DESC_USER: {
1608                 UserCopyOrMarkData data = { copy_func, queue };
1609                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1610                 set_user_copy_or_mark_data (&data);
1611                 marker (start_root, single_arg_user_copy_or_mark);
1612                 set_user_copy_or_mark_data (NULL);
1613                 break;
1614         }
1615         case ROOT_DESC_RUN_LEN:
1616                 g_assert_not_reached ();
1617         default:
1618                 g_assert_not_reached ();
1619         }
1620 }
1621
1622 static void
1623 reset_heap_boundaries (void)
1624 {
1625         lowest_heap_address = ~(mword)0;
1626         highest_heap_address = 0;
1627 }
1628
1629 void
1630 sgen_update_heap_boundaries (mword low, mword high)
1631 {
1632         mword old;
1633
1634         do {
1635                 old = lowest_heap_address;
1636                 if (low >= old)
1637                         break;
1638         } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1639
1640         do {
1641                 old = highest_heap_address;
1642                 if (high <= old)
1643                         break;
1644         } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1645 }
1646
1647 /*
1648  * Allocate and setup the data structures needed to be able to allocate objects
1649  * in the nursery. The nursery is stored in nursery_section.
1650  */
1651 static void
1652 alloc_nursery (void)
1653 {
1654         GCMemSection *section;
1655         char *data;
1656         int scan_starts;
1657         int alloc_size;
1658
1659         if (nursery_section)
1660                 return;
1661         SGEN_LOG (2, "Allocating nursery size: %lu", (unsigned long)sgen_nursery_size);
1662         /* later we will alloc a larger area for the nursery but only activate
1663          * what we need. The rest will be used as expansion if we have too many pinned
1664          * objects in the existing nursery.
1665          */
1666         /* FIXME: handle OOM */
1667         section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1668
1669         alloc_size = sgen_nursery_size;
1670
1671         /* If there isn't enough space even for the nursery we should simply abort. */
1672         g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1673
1674 #ifdef SGEN_ALIGN_NURSERY
1675         data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1676 #else
1677         data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1678 #endif
1679         sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1680         SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1681         section->data = section->next_data = data;
1682         section->size = alloc_size;
1683         section->end_data = data + sgen_nursery_size;
1684         scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1685         section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1686         section->num_scan_start = scan_starts;
1687
1688         nursery_section = section;
1689
1690         sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1691 }
1692
1693 void*
1694 mono_gc_get_nursery (int *shift_bits, size_t *size)
1695 {
1696         *size = sgen_nursery_size;
1697 #ifdef SGEN_ALIGN_NURSERY
1698         *shift_bits = DEFAULT_NURSERY_BITS;
1699 #else
1700         *shift_bits = -1;
1701 #endif
1702         return sgen_get_nursery_start ();
1703 }
1704
1705 void
1706 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1707 {
1708         SgenThreadInfo *info = mono_thread_info_current ();
1709
1710         /* Could be called from sgen_thread_unregister () with a NULL info */
1711         if (domain) {
1712                 g_assert (info);
1713                 info->stopped_domain = domain;
1714         }
1715 }
1716
1717 gboolean
1718 mono_gc_precise_stack_mark_enabled (void)
1719 {
1720         return !conservative_stack_mark;
1721 }
1722
1723 FILE *
1724 mono_gc_get_logfile (void)
1725 {
1726         return gc_debug_file;
1727 }
1728
1729 static void
1730 report_finalizer_roots_list (FinalizeReadyEntry *list)
1731 {
1732         GCRootReport report;
1733         FinalizeReadyEntry *fin;
1734
1735         report.count = 0;
1736         for (fin = list; fin; fin = fin->next) {
1737                 if (!fin->object)
1738                         continue;
1739                 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1740         }
1741         notify_gc_roots (&report);
1742 }
1743
1744 static void
1745 report_finalizer_roots (void)
1746 {
1747         report_finalizer_roots_list (fin_ready_list);
1748         report_finalizer_roots_list (critical_fin_list);
1749 }
1750
1751 static GCRootReport *root_report;
1752
1753 static void
1754 single_arg_report_root (void **obj)
1755 {
1756         if (*obj)
1757                 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1758 }
1759
1760 static void
1761 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1762 {
1763         switch (desc & ROOT_DESC_TYPE_MASK) {
1764         case ROOT_DESC_BITMAP:
1765                 desc >>= ROOT_DESC_TYPE_SHIFT;
1766                 while (desc) {
1767                         if ((desc & 1) && *start_root) {
1768                                 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1769                         }
1770                         desc >>= 1;
1771                         start_root++;
1772                 }
1773                 return;
1774         case ROOT_DESC_COMPLEX: {
1775                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1776                 int bwords = (*bitmap_data) - 1;
1777                 void **start_run = start_root;
1778                 bitmap_data++;
1779                 while (bwords-- > 0) {
1780                         gsize bmap = *bitmap_data++;
1781                         void **objptr = start_run;
1782                         while (bmap) {
1783                                 if ((bmap & 1) && *objptr) {
1784                                         add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1785                                 }
1786                                 bmap >>= 1;
1787                                 ++objptr;
1788                         }
1789                         start_run += GC_BITS_PER_WORD;
1790                 }
1791                 break;
1792         }
1793         case ROOT_DESC_USER: {
1794                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1795                 root_report = report;
1796                 marker (start_root, single_arg_report_root);
1797                 break;
1798         }
1799         case ROOT_DESC_RUN_LEN:
1800                 g_assert_not_reached ();
1801         default:
1802                 g_assert_not_reached ();
1803         }
1804 }
1805
1806 static void
1807 report_registered_roots_by_type (int root_type)
1808 {
1809         GCRootReport report;
1810         void **start_root;
1811         RootRecord *root;
1812         report.count = 0;
1813         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1814                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1815                 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1816         } SGEN_HASH_TABLE_FOREACH_END;
1817         notify_gc_roots (&report);
1818 }
1819
1820 static void
1821 report_registered_roots (void)
1822 {
1823         report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1824         report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1825 }
1826
1827 static void
1828 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1829 {
1830         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1831         SgenGrayQueue *queue = ctx.queue;
1832         FinalizeReadyEntry *fin;
1833
1834         for (fin = list; fin; fin = fin->next) {
1835                 if (!fin->object)
1836                         continue;
1837                 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1838                 copy_func (&fin->object, queue);
1839         }
1840 }
1841
1842 static const char*
1843 generation_name (int generation)
1844 {
1845         switch (generation) {
1846         case GENERATION_NURSERY: return "nursery";
1847         case GENERATION_OLD: return "old";
1848         default: g_assert_not_reached ();
1849         }
1850 }
1851
1852 const char*
1853 sgen_generation_name (int generation)
1854 {
1855         return generation_name (generation);
1856 }
1857
1858 SgenObjectOperations *
1859 sgen_get_current_object_ops (void){
1860         return &current_object_ops;
1861 }
1862
1863
1864 static void
1865 finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue)
1866 {
1867         TV_DECLARE (atv);
1868         TV_DECLARE (btv);
1869         int done_with_ephemerons, ephemeron_rounds = 0;
1870         CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1871         ScanObjectFunc scan_func = current_object_ops.scan_object;
1872         ScanCopyContext ctx = { scan_func, copy_func, queue };
1873
1874         /*
1875          * We copied all the reachable objects. Now it's the time to copy
1876          * the objects that were not referenced by the roots, but by the copied objects.
1877          * we built a stack of objects pointed to by gray_start: they are
1878          * additional roots and we may add more items as we go.
1879          * We loop until gray_start == gray_objects which means no more objects have
1880          * been added. Note this is iterative: no recursion is involved.
1881          * We need to walk the LO list as well in search of marked big objects
1882          * (use a flag since this is needed only on major collections). We need to loop
1883          * here as well, so keep a counter of marked LO (increasing it in copy_object).
1884          *   To achieve better cache locality and cache usage, we drain the gray stack
1885          * frequently, after each object is copied, and just finish the work here.
1886          */
1887         sgen_drain_gray_stack (-1, ctx);
1888         TV_GETTIME (atv);
1889         SGEN_LOG (2, "%s generation done", generation_name (generation));
1890
1891         /*
1892         Reset bridge data, we might have lingering data from a previous collection if this is a major
1893         collection trigged by minor overflow.
1894
1895         We must reset the gathered bridges since their original block might be evacuated due to major
1896         fragmentation in the meanwhile and the bridge code should not have to deal with that.
1897         */
1898         sgen_bridge_reset_data ();
1899
1900         /*
1901          * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1902          * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1903          * objects that are in fact reachable.
1904          */
1905         done_with_ephemerons = 0;
1906         do {
1907                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1908                 sgen_drain_gray_stack (-1, ctx);
1909                 ++ephemeron_rounds;
1910         } while (!done_with_ephemerons);
1911
1912         sgen_scan_togglerefs (start_addr, end_addr, ctx);
1913         if (generation == GENERATION_OLD)
1914                 sgen_scan_togglerefs (sgen_get_nursery_start (), sgen_get_nursery_end (), ctx);
1915
1916         if (sgen_need_bridge_processing ()) {
1917                 sgen_collect_bridge_objects (generation, ctx);
1918                 if (generation == GENERATION_OLD)
1919                         sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1920         }
1921
1922         /*
1923         Make sure we drain the gray stack before processing disappearing links and finalizers.
1924         If we don't make sure it is empty we might wrongly see a live object as dead.
1925         */
1926         sgen_drain_gray_stack (-1, ctx);
1927
1928         /*
1929         We must clear weak links that don't track resurrection before processing object ready for
1930         finalization so they can be cleared before that.
1931         */
1932         sgen_null_link_in_range (generation, TRUE, ctx);
1933         if (generation == GENERATION_OLD)
1934                 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1935
1936
1937         /* walk the finalization queue and move also the objects that need to be
1938          * finalized: use the finalized objects as new roots so the objects they depend
1939          * on are also not reclaimed. As with the roots above, only objects in the nursery
1940          * are marked/copied.
1941          */
1942         sgen_finalize_in_range (generation, ctx);
1943         if (generation == GENERATION_OLD)
1944                 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1945         /* drain the new stack that might have been created */
1946         SGEN_LOG (6, "Precise scan of gray area post fin");
1947         sgen_drain_gray_stack (-1, ctx);
1948
1949         /*
1950          * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1951          */
1952         done_with_ephemerons = 0;
1953         do {
1954                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1955                 sgen_drain_gray_stack (-1, ctx);
1956                 ++ephemeron_rounds;
1957         } while (!done_with_ephemerons);
1958
1959         /*
1960          * Clear ephemeron pairs with unreachable keys.
1961          * We pass the copy func so we can figure out if an array was promoted or not.
1962          */
1963         clear_unreachable_ephemerons (ctx);
1964
1965         TV_GETTIME (btv);
1966         SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
1967
1968         /*
1969          * handle disappearing links
1970          * Note we do this after checking the finalization queue because if an object
1971          * survives (at least long enough to be finalized) we don't clear the link.
1972          * This also deals with a possible issue with the monitor reclamation: with the Boehm
1973          * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1974          * called.
1975          */
1976         g_assert (sgen_gray_object_queue_is_empty (queue));
1977         for (;;) {
1978                 sgen_null_link_in_range (generation, FALSE, ctx);
1979                 if (generation == GENERATION_OLD)
1980                         sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
1981                 if (sgen_gray_object_queue_is_empty (queue))
1982                         break;
1983                 sgen_drain_gray_stack (-1, ctx);
1984         }
1985
1986         g_assert (sgen_gray_object_queue_is_empty (queue));
1987 }
1988
1989 void
1990 sgen_check_section_scan_starts (GCMemSection *section)
1991 {
1992         int i;
1993         for (i = 0; i < section->num_scan_start; ++i) {
1994                 if (section->scan_starts [i]) {
1995                         guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
1996                         g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
1997                 }
1998         }
1999 }
2000
2001 static void
2002 check_scan_starts (void)
2003 {
2004         if (!do_scan_starts_check)
2005                 return;
2006         sgen_check_section_scan_starts (nursery_section);
2007         major_collector.check_scan_starts ();
2008 }
2009
2010 static void
2011 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
2012 {
2013         void **start_root;
2014         RootRecord *root;
2015         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
2016                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
2017                 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
2018         } SGEN_HASH_TABLE_FOREACH_END;
2019 }
2020
2021 void
2022 sgen_dump_occupied (char *start, char *end, char *section_start)
2023 {
2024         fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2025 }
2026
2027 void
2028 sgen_dump_section (GCMemSection *section, const char *type)
2029 {
2030         char *start = section->data;
2031         char *end = section->data + section->size;
2032         char *occ_start = NULL;
2033         GCVTable *vt;
2034         char *old_start = NULL; /* just for debugging */
2035
2036         fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2037
2038         while (start < end) {
2039                 guint size;
2040                 MonoClass *class;
2041
2042                 if (!*(void**)start) {
2043                         if (occ_start) {
2044                                 sgen_dump_occupied (occ_start, start, section->data);
2045                                 occ_start = NULL;
2046                         }
2047                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2048                         continue;
2049                 }
2050                 g_assert (start < section->next_data);
2051
2052                 if (!occ_start)
2053                         occ_start = start;
2054
2055                 vt = (GCVTable*)LOAD_VTABLE (start);
2056                 class = vt->klass;
2057
2058                 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2059
2060                 /*
2061                 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2062                                 start - section->data,
2063                                 vt->klass->name_space, vt->klass->name,
2064                                 size);
2065                 */
2066
2067                 old_start = start;
2068                 start += size;
2069         }
2070         if (occ_start)
2071                 sgen_dump_occupied (occ_start, start, section->data);
2072
2073         fprintf (heap_dump_file, "</section>\n");
2074 }
2075
2076 static void
2077 dump_object (MonoObject *obj, gboolean dump_location)
2078 {
2079         static char class_name [1024];
2080
2081         MonoClass *class = mono_object_class (obj);
2082         int i, j;
2083
2084         /*
2085          * Python's XML parser is too stupid to parse angle brackets
2086          * in strings, so we just ignore them;
2087          */
2088         i = j = 0;
2089         while (class->name [i] && j < sizeof (class_name) - 1) {
2090                 if (!strchr ("<>\"", class->name [i]))
2091                         class_name [j++] = class->name [i];
2092                 ++i;
2093         }
2094         g_assert (j < sizeof (class_name));
2095         class_name [j] = 0;
2096
2097         fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2098                         class->name_space, class_name,
2099                         safe_object_get_size (obj));
2100         if (dump_location) {
2101                 const char *location;
2102                 if (ptr_in_nursery (obj))
2103                         location = "nursery";
2104                 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2105                         location = "major";
2106                 else
2107                         location = "LOS";
2108                 fprintf (heap_dump_file, " location=\"%s\"", location);
2109         }
2110         fprintf (heap_dump_file, "/>\n");
2111 }
2112
2113 static void
2114 dump_heap (const char *type, int num, const char *reason)
2115 {
2116         ObjectList *list;
2117         LOSObject *bigobj;
2118
2119         fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2120         if (reason)
2121                 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2122         fprintf (heap_dump_file, ">\n");
2123         fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2124         sgen_dump_internal_mem_usage (heap_dump_file);
2125         fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
2126         /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2127         fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
2128
2129         fprintf (heap_dump_file, "<pinned-objects>\n");
2130         for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
2131                 dump_object (list->obj, TRUE);
2132         fprintf (heap_dump_file, "</pinned-objects>\n");
2133
2134         sgen_dump_section (nursery_section, "nursery");
2135
2136         major_collector.dump_heap (heap_dump_file);
2137
2138         fprintf (heap_dump_file, "<los>\n");
2139         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2140                 dump_object ((MonoObject*)bigobj->data, FALSE);
2141         fprintf (heap_dump_file, "</los>\n");
2142
2143         fprintf (heap_dump_file, "</collection>\n");
2144 }
2145
2146 void
2147 sgen_register_moved_object (void *obj, void *destination)
2148 {
2149         g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
2150
2151         /* FIXME: handle this for parallel collector */
2152         g_assert (!sgen_collection_is_parallel ());
2153
2154         if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2155                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2156                 moved_objects_idx = 0;
2157         }
2158         moved_objects [moved_objects_idx++] = obj;
2159         moved_objects [moved_objects_idx++] = destination;
2160 }
2161
2162 static void
2163 init_stats (void)
2164 {
2165         static gboolean inited = FALSE;
2166
2167         if (inited)
2168                 return;
2169
2170         mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pre_collection_fragment_clear);
2171         mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pinning);
2172         mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_remsets);
2173         mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_pinned);
2174         mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_registered_roots);
2175         mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_thread_data);
2176         mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_finish_gray_stack);
2177         mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_fragment_creation);
2178
2179         mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pre_collection_fragment_clear);
2180         mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pinning);
2181         mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_pinned);
2182         mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_registered_roots);
2183         mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_thread_data);
2184         mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_alloc_pinned);
2185         mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_finalized);
2186         mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_big_objects);
2187         mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_finish_gray_stack);
2188         mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_free_bigobjs);
2189         mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_los_sweep);
2190         mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_sweep);
2191         mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_fragment_creation);
2192
2193         mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_pinned_objects);
2194
2195 #ifdef HEAVY_STATISTICS
2196         mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
2197         mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
2198         mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
2199         mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
2200         mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
2201         mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
2202         mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
2203
2204         mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
2205         mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
2206
2207         mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
2208         mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
2209         mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
2210         mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
2211
2212         mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
2213         mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
2214
2215         mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
2216
2217         mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
2218         mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
2219         mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
2220         mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_to_space);
2221
2222         sgen_nursery_allocator_init_heavy_stats ();
2223         sgen_alloc_init_heavy_stats ();
2224 #endif
2225
2226         inited = TRUE;
2227 }
2228
2229
2230 static void
2231 reset_pinned_from_failed_allocation (void)
2232 {
2233         bytes_pinned_from_failed_allocation = 0;
2234 }
2235
2236 void
2237 sgen_set_pinned_from_failed_allocation (mword objsize)
2238 {
2239         bytes_pinned_from_failed_allocation += objsize;
2240 }
2241
2242 gboolean
2243 sgen_collection_is_parallel (void)
2244 {
2245         switch (current_collection_generation) {
2246         case GENERATION_NURSERY:
2247                 return nursery_collection_is_parallel;
2248         case GENERATION_OLD:
2249                 return major_collector.is_parallel;
2250         default:
2251                 g_error ("Invalid current generation %d", current_collection_generation);
2252         }
2253 }
2254
2255 gboolean
2256 sgen_collection_is_concurrent (void)
2257 {
2258         switch (current_collection_generation) {
2259         case GENERATION_NURSERY:
2260                 return FALSE;
2261         case GENERATION_OLD:
2262                 return concurrent_collection_in_progress;
2263         default:
2264                 g_error ("Invalid current generation %d", current_collection_generation);
2265         }
2266 }
2267
2268 gboolean
2269 sgen_concurrent_collection_in_progress (void)
2270 {
2271         return concurrent_collection_in_progress;
2272 }
2273
2274 typedef struct
2275 {
2276         char *heap_start;
2277         char *heap_end;
2278 } FinishRememberedSetScanJobData;
2279
2280 static void
2281 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2282 {
2283         FinishRememberedSetScanJobData *job_data = job_data_untyped;
2284
2285         remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2286         sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2287 }
2288
2289 typedef struct
2290 {
2291         CopyOrMarkObjectFunc copy_or_mark_func;
2292         ScanObjectFunc scan_func;
2293         char *heap_start;
2294         char *heap_end;
2295         int root_type;
2296 } ScanFromRegisteredRootsJobData;
2297
2298 static void
2299 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2300 {
2301         ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2302         ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2303                 sgen_workers_get_job_gray_queue (worker_data) };
2304
2305         scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2306         sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2307 }
2308
2309 typedef struct
2310 {
2311         char *heap_start;
2312         char *heap_end;
2313 } ScanThreadDataJobData;
2314
2315 static void
2316 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2317 {
2318         ScanThreadDataJobData *job_data = job_data_untyped;
2319
2320         scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2321                         sgen_workers_get_job_gray_queue (worker_data));
2322         sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2323 }
2324
2325 typedef struct
2326 {
2327         FinalizeReadyEntry *list;
2328 } ScanFinalizerEntriesJobData;
2329
2330 static void
2331 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2332 {
2333         ScanFinalizerEntriesJobData *job_data = job_data_untyped;
2334         ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2335
2336         scan_finalizer_entries (job_data->list, ctx);
2337         sgen_free_internal_dynamic (job_data, sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2338 }
2339
2340 static void
2341 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2342 {
2343         g_assert (concurrent_collection_in_progress);
2344         major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2345 }
2346
2347 static void
2348 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2349 {
2350         g_assert (concurrent_collection_in_progress);
2351         sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2352 }
2353
2354 static void
2355 verify_scan_starts (char *start, char *end)
2356 {
2357         int i;
2358
2359         for (i = 0; i < nursery_section->num_scan_start; ++i) {
2360                 char *addr = nursery_section->scan_starts [i];
2361                 if (addr > start && addr < end)
2362                         SGEN_LOG (1, "NFC-BAD SCAN START [%d] %p for obj [%p %p]", i, addr, start, end);
2363         }
2364 }
2365
2366 static void
2367 verify_nursery (void)
2368 {
2369         char *start, *end, *cur, *hole_start;
2370
2371         if (!do_verify_nursery)
2372                 return;
2373
2374         /*This cleans up unused fragments */
2375         sgen_nursery_allocator_prepare_for_pinning ();
2376
2377         hole_start = start = cur = sgen_get_nursery_start ();
2378         end = sgen_get_nursery_end ();
2379
2380         while (cur < end) {
2381                 size_t ss, size;
2382
2383                 if (!*(void**)cur) {
2384                         cur += sizeof (void*);
2385                         continue;
2386                 }
2387
2388                 if (object_is_forwarded (cur))
2389                         SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2390                 else if (object_is_pinned (cur))
2391                         SGEN_LOG (1, "PINNED OBJ %p", cur);
2392
2393                 ss = safe_object_get_size ((MonoObject*)cur);
2394                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2395                 verify_scan_starts (cur, cur + size);
2396                 if (do_dump_nursery_content) {
2397                         if (cur > hole_start)
2398                                 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2399                         SGEN_LOG (1, "OBJ  [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2400                 }
2401                 cur += size;
2402                 hole_start = cur;
2403         }
2404 }
2405
2406 /*
2407  * Checks that no objects in the nursery are fowarded or pinned.  This
2408  * is a precondition to restarting the mutator while doing a
2409  * concurrent collection.  Note that we don't clear fragments because
2410  * we depend on that having happened earlier.
2411  */
2412 static void
2413 check_nursery_is_clean (void)
2414 {
2415         char *start, *end, *cur;
2416
2417         start = cur = sgen_get_nursery_start ();
2418         end = sgen_get_nursery_end ();
2419
2420         while (cur < end) {
2421                 size_t ss, size;
2422
2423                 if (!*(void**)cur) {
2424                         cur += sizeof (void*);
2425                         continue;
2426                 }
2427
2428                 g_assert (!object_is_forwarded (cur));
2429                 g_assert (!object_is_pinned (cur));
2430
2431                 ss = safe_object_get_size ((MonoObject*)cur);
2432                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2433                 verify_scan_starts (cur, cur + size);
2434
2435                 cur += size;
2436         }
2437 }
2438
2439 static void
2440 init_gray_queue (void)
2441 {
2442         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) {
2443                 sgen_workers_init_distribute_gray_queue ();
2444                 sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
2445                                 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2446         } else {
2447                 sgen_gray_object_queue_init (&gray_queue, NULL);
2448         }
2449 }
2450
2451 static void
2452 pin_stage_object_callback (char *obj, size_t size, void *data)
2453 {
2454         sgen_pin_stage_ptr (obj);
2455         /* FIXME: do pin stats if enabled */
2456 }
2457
2458 /*
2459  * Collect objects in the nursery.  Returns whether to trigger a major
2460  * collection.
2461  */
2462 static gboolean
2463 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2464 {
2465         gboolean needs_major;
2466         size_t max_garbage_amount;
2467         char *nursery_next;
2468         FinishRememberedSetScanJobData *frssjd;
2469         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2470         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2471         ScanThreadDataJobData *stdjd;
2472         mword fragment_total;
2473         ScanCopyContext ctx;
2474         TV_DECLARE (all_atv);
2475         TV_DECLARE (all_btv);
2476         TV_DECLARE (atv);
2477         TV_DECLARE (btv);
2478
2479         if (disable_minor_collections)
2480                 return TRUE;
2481
2482         MONO_GC_BEGIN (GENERATION_NURSERY);
2483         binary_protocol_collection_begin (stat_minor_gcs, GENERATION_NURSERY);
2484
2485         verify_nursery ();
2486
2487 #ifndef DISABLE_PERFCOUNTERS
2488         mono_perfcounters->gc_collections0++;
2489 #endif
2490
2491         current_collection_generation = GENERATION_NURSERY;
2492         if (sgen_collection_is_parallel ())
2493                 current_object_ops = sgen_minor_collector.parallel_ops;
2494         else
2495                 current_object_ops = sgen_minor_collector.serial_ops;
2496
2497         reset_pinned_from_failed_allocation ();
2498
2499         check_scan_starts ();
2500
2501         sgen_nursery_alloc_prepare_for_minor ();
2502
2503         degraded_mode = 0;
2504         objects_pinned = 0;
2505         nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2506         /* FIXME: optimize later to use the higher address where an object can be present */
2507         nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2508
2509         SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", stat_minor_gcs, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2510         max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2511         g_assert (nursery_section->size >= max_garbage_amount);
2512
2513         /* world must be stopped already */
2514         TV_GETTIME (all_atv);
2515         atv = all_atv;
2516
2517         TV_GETTIME (btv);
2518         time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2519
2520         if (xdomain_checks) {
2521                 sgen_clear_nursery_fragments ();
2522                 check_for_xdomain_refs ();
2523         }
2524
2525         nursery_section->next_data = nursery_next;
2526
2527         major_collector.start_nursery_collection ();
2528
2529         sgen_memgov_minor_collection_start ();
2530
2531         init_gray_queue ();
2532
2533         stat_minor_gcs++;
2534         gc_stats.minor_gc_count ++;
2535
2536         if (remset.prepare_for_minor_collection)
2537                 remset.prepare_for_minor_collection ();
2538
2539         MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2540
2541         sgen_process_fin_stage_entries ();
2542         sgen_process_dislink_stage_entries ();
2543
2544         MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2545
2546         /* pin from pinned handles */
2547         sgen_init_pinning ();
2548         mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2549         pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2550         /* pin cemented objects */
2551         sgen_cement_iterate (pin_stage_object_callback, NULL);
2552         /* identify pinned objects */
2553         sgen_optimize_pin_queue (0);
2554         sgen_pinning_setup_section (nursery_section);
2555         ctx.scan_func = NULL;
2556         ctx.copy_func = NULL;
2557         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2558         sgen_pin_objects_in_section (nursery_section, ctx);
2559         sgen_pinning_trim_queue_to_section (nursery_section);
2560
2561         TV_GETTIME (atv);
2562         time_minor_pinning += TV_ELAPSED (btv, atv);
2563         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2564         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2565
2566         MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2567
2568         if (whole_heap_check_before_collection) {
2569                 sgen_clear_nursery_fragments ();
2570                 sgen_check_whole_heap (finish_up_concurrent_mark);
2571         }
2572         if (consistency_check_at_minor_collection)
2573                 sgen_check_consistency ();
2574
2575         sgen_workers_start_all_workers ();
2576
2577         /*
2578          * Perform the sequential part of remembered set scanning.
2579          * This usually involves scanning global information that might later be produced by evacuation.
2580          */
2581         if (remset.begin_scan_remsets)
2582                 remset.begin_scan_remsets (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2583
2584         sgen_workers_start_marking ();
2585
2586         frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2587         frssjd->heap_start = sgen_get_nursery_start ();
2588         frssjd->heap_end = nursery_next;
2589         sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2590
2591         /* we don't have complete write barrier yet, so we scan all the old generation sections */
2592         TV_GETTIME (btv);
2593         time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2594         SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2595
2596         MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2597
2598         if (!sgen_collection_is_parallel ()) {
2599                 ctx.scan_func = current_object_ops.scan_object;
2600                 ctx.copy_func = NULL;
2601                 ctx.queue = &gray_queue;
2602                 sgen_drain_gray_stack (-1, ctx);
2603         }
2604
2605         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2606                 report_registered_roots ();
2607         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2608                 report_finalizer_roots ();
2609         TV_GETTIME (atv);
2610         time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2611
2612         MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2613
2614         /* registered roots, this includes static fields */
2615         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2616         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2617         scrrjd_normal->scan_func = current_object_ops.scan_object;
2618         scrrjd_normal->heap_start = sgen_get_nursery_start ();
2619         scrrjd_normal->heap_end = nursery_next;
2620         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2621         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2622
2623         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2624         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2625         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2626         scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2627         scrrjd_wbarrier->heap_end = nursery_next;
2628         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2629         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2630
2631         TV_GETTIME (btv);
2632         time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2633
2634         MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2635
2636         /* thread data */
2637         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2638         stdjd->heap_start = sgen_get_nursery_start ();
2639         stdjd->heap_end = nursery_next;
2640         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2641
2642         TV_GETTIME (atv);
2643         time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2644         btv = atv;
2645
2646         MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2647
2648         g_assert (!sgen_collection_is_parallel () && !sgen_collection_is_concurrent ());
2649
2650         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ())
2651                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2652
2653         /* Scan the list of objects ready for finalization. If */
2654         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2655         sfejd_fin_ready->list = fin_ready_list;
2656         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
2657
2658         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2659         sfejd_critical_fin->list = critical_fin_list;
2660         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
2661
2662         MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2663
2664         finish_gray_stack (sgen_get_nursery_start (), nursery_next, GENERATION_NURSERY, &gray_queue);
2665         TV_GETTIME (atv);
2666         time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2667         mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2668
2669         MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2670
2671         /*
2672          * The (single-threaded) finalization code might have done
2673          * some copying/marking so we can only reset the GC thread's
2674          * worker data here instead of earlier when we joined the
2675          * workers.
2676          */
2677         sgen_workers_reset_data ();
2678
2679         if (objects_pinned) {
2680                 sgen_optimize_pin_queue (0);
2681                 sgen_pinning_setup_section (nursery_section);
2682         }
2683
2684         /* walk the pin_queue, build up the fragment list of free memory, unmark
2685          * pinned objects as we go, memzero() the empty fragments so they are ready for the
2686          * next allocations.
2687          */
2688         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2689         fragment_total = sgen_build_nursery_fragments (nursery_section,
2690                         nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries,
2691                         unpin_queue);
2692         if (!fragment_total)
2693                 degraded_mode = 1;
2694
2695         /* Clear TLABs for all threads */
2696         sgen_clear_tlabs ();
2697
2698         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2699         TV_GETTIME (btv);
2700         time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2701         SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2702
2703         if (consistency_check_at_minor_collection)
2704                 sgen_check_major_refs ();
2705
2706         major_collector.finish_nursery_collection ();
2707
2708         TV_GETTIME (all_btv);
2709         gc_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2710
2711         if (heap_dump_file)
2712                 dump_heap ("minor", stat_minor_gcs - 1, NULL);
2713
2714         /* prepare the pin queue for the next collection */
2715         sgen_finish_pinning ();
2716         if (fin_ready_list || critical_fin_list) {
2717                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2718                 mono_gc_finalize_notify ();
2719         }
2720         sgen_pin_stats_reset ();
2721         /* clear cemented hash */
2722         sgen_cement_clear_below_threshold ();
2723
2724         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2725
2726         if (remset.finish_minor_collection)
2727                 remset.finish_minor_collection ();
2728
2729         check_scan_starts ();
2730
2731         binary_protocol_flush_buffers (FALSE);
2732
2733         sgen_memgov_minor_collection_end ();
2734
2735         /*objects are late pinned because of lack of memory, so a major is a good call*/
2736         needs_major = objects_pinned > 0;
2737         current_collection_generation = -1;
2738         objects_pinned = 0;
2739
2740         MONO_GC_END (GENERATION_NURSERY);
2741         binary_protocol_collection_end (stat_minor_gcs - 1, GENERATION_NURSERY);
2742
2743         if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2744                 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2745
2746         return needs_major;
2747 }
2748
2749 static void
2750 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2751 {
2752         ctx->scan_func (obj, ctx->queue);
2753 }
2754
2755 static void
2756 scan_nursery_objects (ScanCopyContext ctx)
2757 {
2758         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2759                         (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2760 }
2761
2762 static void
2763 major_copy_or_mark_from_roots (int *old_next_pin_slot, gboolean finish_up_concurrent_mark, gboolean scan_mod_union)
2764 {
2765         LOSObject *bigobj;
2766         TV_DECLARE (atv);
2767         TV_DECLARE (btv);
2768         /* FIXME: only use these values for the precise scan
2769          * note that to_space pointers should be excluded anyway...
2770          */
2771         char *heap_start = NULL;
2772         char *heap_end = (char*)-1;
2773         gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2774         GCRootReport root_report = { 0 };
2775         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2776         ScanThreadDataJobData *stdjd;
2777         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2778         ScanCopyContext ctx;
2779
2780         if (concurrent_collection_in_progress) {
2781                 /*This cleans up unused fragments */
2782                 sgen_nursery_allocator_prepare_for_pinning ();
2783
2784                 if (do_concurrent_checks)
2785                         check_nursery_is_clean ();
2786         } else {
2787                 /* The concurrent collector doesn't touch the nursery. */
2788                 sgen_nursery_alloc_prepare_for_major ();
2789         }
2790
2791         init_gray_queue ();
2792
2793         TV_GETTIME (atv);
2794
2795         /* Pinning depends on this */
2796         sgen_clear_nursery_fragments ();
2797
2798         if (whole_heap_check_before_collection)
2799                 sgen_check_whole_heap (finish_up_concurrent_mark);
2800
2801         TV_GETTIME (btv);
2802         time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2803
2804         if (!sgen_collection_is_concurrent ())
2805                 nursery_section->next_data = sgen_get_nursery_end ();
2806         /* we should also coalesce scanning from sections close to each other
2807          * and deal with pointers outside of the sections later.
2808          */
2809
2810         objects_pinned = 0;
2811         *major_collector.have_swept = FALSE;
2812
2813         if (xdomain_checks) {
2814                 sgen_clear_nursery_fragments ();
2815                 check_for_xdomain_refs ();
2816         }
2817
2818         if (!concurrent_collection_in_progress) {
2819                 /* Remsets are not useful for a major collection */
2820                 remset.prepare_for_major_collection ();
2821         }
2822
2823         sgen_process_fin_stage_entries ();
2824         sgen_process_dislink_stage_entries ();
2825
2826         TV_GETTIME (atv);
2827         sgen_init_pinning ();
2828         SGEN_LOG (6, "Collecting pinned addresses");
2829         pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2830
2831         if (!concurrent_collection_in_progress) {
2832                 if (major_collector.is_concurrent) {
2833                         /*
2834                          * The concurrent major collector cannot evict
2835                          * yet, so we need to pin cemented objects to
2836                          * not break some asserts.
2837                          */
2838                         sgen_cement_iterate (pin_stage_object_callback, NULL);
2839                 }
2840
2841                 sgen_cement_reset ();
2842         }
2843
2844         sgen_optimize_pin_queue (0);
2845
2846         /*
2847          * The concurrent collector doesn't move objects, neither on
2848          * the major heap nor in the nursery, so we can mark even
2849          * before pinning has finished.  For the non-concurrent
2850          * collector we start the workers after pinning.
2851          */
2852         if (concurrent_collection_in_progress) {
2853                 sgen_workers_start_all_workers ();
2854                 sgen_workers_start_marking ();
2855         }
2856
2857         /*
2858          * pin_queue now contains all candidate pointers, sorted and
2859          * uniqued.  We must do two passes now to figure out which
2860          * objects are pinned.
2861          *
2862          * The first is to find within the pin_queue the area for each
2863          * section.  This requires that the pin_queue be sorted.  We
2864          * also process the LOS objects and pinned chunks here.
2865          *
2866          * The second, destructive, pass is to reduce the section
2867          * areas to pointers to the actually pinned objects.
2868          */
2869         SGEN_LOG (6, "Pinning from sections");
2870         /* first pass for the sections */
2871         sgen_find_section_pin_queue_start_end (nursery_section);
2872         major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2873         /* identify possible pointers to the insize of large objects */
2874         SGEN_LOG (6, "Pinning from large objects");
2875         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2876                 int dummy;
2877                 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) {
2878                         binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2879
2880 #ifdef ENABLE_DTRACE
2881                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2882                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2883                                 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2884                         }
2885 #endif
2886
2887                         if (sgen_los_object_is_pinned (bigobj->data)) {
2888                                 g_assert (finish_up_concurrent_mark);
2889                                 continue;
2890                         }
2891                         sgen_los_pin_object (bigobj->data);
2892                         /* FIXME: only enqueue if object has references */
2893                         GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data);
2894                         if (G_UNLIKELY (do_pin_stats))
2895                                 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2896                         SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2897
2898                         if (profile_roots)
2899                                 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2900                 }
2901         }
2902         if (profile_roots)
2903                 notify_gc_roots (&root_report);
2904         /* second pass for the sections */
2905         ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2906         ctx.copy_func = NULL;
2907         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2908
2909         /*
2910          * Concurrent mark never follows references into the nursery.
2911          * In the start and finish pauses we must scan live nursery
2912          * objects, though.  We could simply scan all nursery objects,
2913          * but that would be conservative.  The easiest way is to do a
2914          * nursery collection, which copies all live nursery objects
2915          * (except pinned ones, with the simple nursery) to the major
2916          * heap.  Scanning the mod union table later will then scan
2917          * those promoted objects, provided they're reachable.  Pinned
2918          * objects in the nursery - which we can trivially find in the
2919          * pinning queue - are treated as roots in the mark pauses.
2920          *
2921          * The split nursery complicates the latter part because
2922          * non-pinned objects can survive in the nursery.  That's why
2923          * we need to do a full front-to-back scan of the nursery,
2924          * marking all objects.
2925          *
2926          * Non-concurrent mark evacuates from the nursery, so it's
2927          * sufficient to just scan pinned nursery objects.
2928          */
2929         if (concurrent_collection_in_progress && sgen_minor_collector.is_split) {
2930                 scan_nursery_objects (ctx);
2931         } else {
2932                 sgen_pin_objects_in_section (nursery_section, ctx);
2933                 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2934                         sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2935         }
2936
2937         major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2938         if (old_next_pin_slot)
2939                 *old_next_pin_slot = sgen_get_pinned_count ();
2940
2941         TV_GETTIME (btv);
2942         time_major_pinning += TV_ELAPSED (atv, btv);
2943         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2944         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2945
2946         major_collector.init_to_space ();
2947
2948 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2949         main_gc_thread = mono_native_thread_self ();
2950 #endif
2951
2952         if (!concurrent_collection_in_progress && major_collector.is_parallel) {
2953                 sgen_workers_start_all_workers ();
2954                 sgen_workers_start_marking ();
2955         }
2956
2957         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2958                 report_registered_roots ();
2959         TV_GETTIME (atv);
2960         time_major_scan_pinned += TV_ELAPSED (btv, atv);
2961
2962         /* registered roots, this includes static fields */
2963         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2964         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2965         scrrjd_normal->scan_func = current_object_ops.scan_object;
2966         scrrjd_normal->heap_start = heap_start;
2967         scrrjd_normal->heap_end = heap_end;
2968         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2969         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2970
2971         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2972         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2973         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2974         scrrjd_wbarrier->heap_start = heap_start;
2975         scrrjd_wbarrier->heap_end = heap_end;
2976         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2977         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2978
2979         TV_GETTIME (btv);
2980         time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
2981
2982         /* Threads */
2983         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2984         stdjd->heap_start = heap_start;
2985         stdjd->heap_end = heap_end;
2986         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2987
2988         TV_GETTIME (atv);
2989         time_major_scan_thread_data += TV_ELAPSED (btv, atv);
2990
2991         TV_GETTIME (btv);
2992         time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
2993
2994         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2995                 report_finalizer_roots ();
2996
2997         /* scan the list of objects ready for finalization */
2998         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2999         sfejd_fin_ready->list = fin_ready_list;
3000         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
3001
3002         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3003         sfejd_critical_fin->list = critical_fin_list;
3004         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
3005
3006         if (scan_mod_union) {
3007                 g_assert (finish_up_concurrent_mark);
3008
3009                 /* Mod union card table */
3010                 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
3011                 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
3012         }
3013
3014         TV_GETTIME (atv);
3015         time_major_scan_finalized += TV_ELAPSED (btv, atv);
3016         SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
3017
3018         TV_GETTIME (btv);
3019         time_major_scan_big_objects += TV_ELAPSED (atv, btv);
3020
3021         if (concurrent_collection_in_progress) {
3022                 /* prepare the pin queue for the next collection */
3023                 sgen_finish_pinning ();
3024
3025                 sgen_pin_stats_reset ();
3026
3027                 if (do_concurrent_checks)
3028                         check_nursery_is_clean ();
3029         }
3030 }
3031
3032 static void
3033 major_start_collection (gboolean concurrent, int *old_next_pin_slot)
3034 {
3035         MONO_GC_BEGIN (GENERATION_OLD);
3036         binary_protocol_collection_begin (stat_major_gcs, GENERATION_OLD);
3037
3038         current_collection_generation = GENERATION_OLD;
3039 #ifndef DISABLE_PERFCOUNTERS
3040         mono_perfcounters->gc_collections1++;
3041 #endif
3042
3043         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3044
3045         if (concurrent) {
3046                 g_assert (major_collector.is_concurrent);
3047                 concurrent_collection_in_progress = TRUE;
3048
3049                 sgen_cement_concurrent_start ();
3050         }
3051
3052         current_object_ops = major_collector.major_ops;
3053
3054         reset_pinned_from_failed_allocation ();
3055
3056         sgen_memgov_major_collection_start ();
3057
3058         //count_ref_nonref_objs ();
3059         //consistency_check ();
3060
3061         check_scan_starts ();
3062
3063         degraded_mode = 0;
3064         SGEN_LOG (1, "Start major collection %d", stat_major_gcs);
3065         stat_major_gcs++;
3066         gc_stats.major_gc_count ++;
3067
3068         if (major_collector.start_major_collection)
3069                 major_collector.start_major_collection ();
3070
3071         major_copy_or_mark_from_roots (old_next_pin_slot, FALSE, FALSE);
3072 }
3073
3074 static void
3075 wait_for_workers_to_finish (void)
3076 {
3077         if (concurrent_collection_in_progress || major_collector.is_parallel) {
3078                 gray_queue_redirect (&gray_queue);
3079                 sgen_workers_join ();
3080         }
3081
3082         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3083
3084 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
3085         main_gc_thread = NULL;
3086 #endif
3087 }
3088
3089 static void
3090 major_finish_collection (const char *reason, int old_next_pin_slot, gboolean scan_mod_union)
3091 {
3092         LOSObject *bigobj, *prevbo;
3093         TV_DECLARE (atv);
3094         TV_DECLARE (btv);
3095         char *heap_start = NULL;
3096         char *heap_end = (char*)-1;
3097
3098         TV_GETTIME (btv);
3099
3100         if (concurrent_collection_in_progress || major_collector.is_parallel)
3101                 wait_for_workers_to_finish ();
3102
3103         current_object_ops = major_collector.major_ops;
3104
3105         if (concurrent_collection_in_progress) {
3106                 major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
3107                 wait_for_workers_to_finish ();
3108
3109                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3110
3111                 if (do_concurrent_checks)
3112                         check_nursery_is_clean ();
3113         }
3114
3115         /*
3116          * The workers have stopped so we need to finish gray queue
3117          * work that might result from finalization in the main GC
3118          * thread.  Redirection must therefore be turned off.
3119          */
3120         sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
3121         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3122
3123         /* all the objects in the heap */
3124         finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue);
3125         TV_GETTIME (atv);
3126         time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
3127
3128         /*
3129          * The (single-threaded) finalization code might have done
3130          * some copying/marking so we can only reset the GC thread's
3131          * worker data here instead of earlier when we joined the
3132          * workers.
3133          */
3134         sgen_workers_reset_data ();
3135
3136         if (objects_pinned) {
3137                 g_assert (!concurrent_collection_in_progress);
3138
3139                 /*This is slow, but we just OOM'd*/
3140                 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
3141                 sgen_optimize_pin_queue (0);
3142                 sgen_find_section_pin_queue_start_end (nursery_section);
3143                 objects_pinned = 0;
3144         }
3145
3146         reset_heap_boundaries ();
3147         sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
3148
3149         if (check_mark_bits_after_major_collection)
3150                 sgen_check_major_heap_marked ();
3151
3152         MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
3153
3154         /* sweep the big objects list */
3155         prevbo = NULL;
3156         for (bigobj = los_object_list; bigobj;) {
3157                 g_assert (!object_is_pinned (bigobj->data));
3158                 if (sgen_los_object_is_pinned (bigobj->data)) {
3159                         sgen_los_unpin_object (bigobj->data);
3160                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
3161                 } else {
3162                         LOSObject *to_free;
3163                         /* not referenced anywhere, so we can free it */
3164                         if (prevbo)
3165                                 prevbo->next = bigobj->next;
3166                         else
3167                                 los_object_list = bigobj->next;
3168                         to_free = bigobj;
3169                         bigobj = bigobj->next;
3170                         sgen_los_free_object (to_free);
3171                         continue;
3172                 }
3173                 prevbo = bigobj;
3174                 bigobj = bigobj->next;
3175         }
3176
3177         TV_GETTIME (btv);
3178         time_major_free_bigobjs += TV_ELAPSED (atv, btv);
3179
3180         sgen_los_sweep ();
3181
3182         TV_GETTIME (atv);
3183         time_major_los_sweep += TV_ELAPSED (btv, atv);
3184
3185         major_collector.sweep ();
3186
3187         MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
3188
3189         TV_GETTIME (btv);
3190         time_major_sweep += TV_ELAPSED (atv, btv);
3191
3192         if (!concurrent_collection_in_progress) {
3193                 /* walk the pin_queue, build up the fragment list of free memory, unmark
3194                  * pinned objects as we go, memzero() the empty fragments so they are ready for the
3195                  * next allocations.
3196                  */
3197                 if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries, NULL))
3198                         degraded_mode = 1;
3199
3200                 /* prepare the pin queue for the next collection */
3201                 sgen_finish_pinning ();
3202
3203                 /* Clear TLABs for all threads */
3204                 sgen_clear_tlabs ();
3205
3206                 sgen_pin_stats_reset ();
3207         }
3208
3209         if (concurrent_collection_in_progress)
3210                 sgen_cement_concurrent_finish ();
3211         sgen_cement_clear_below_threshold ();
3212
3213         TV_GETTIME (atv);
3214         time_major_fragment_creation += TV_ELAPSED (btv, atv);
3215
3216         if (heap_dump_file)
3217                 dump_heap ("major", stat_major_gcs - 1, reason);
3218
3219         if (fin_ready_list || critical_fin_list) {
3220                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
3221                 mono_gc_finalize_notify ();
3222         }
3223
3224         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3225
3226         sgen_memgov_major_collection_end ();
3227         current_collection_generation = -1;
3228
3229         major_collector.finish_major_collection ();
3230
3231         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3232
3233         if (concurrent_collection_in_progress)
3234                 concurrent_collection_in_progress = FALSE;
3235
3236         check_scan_starts ();
3237
3238         binary_protocol_flush_buffers (FALSE);
3239
3240         //consistency_check ();
3241
3242         MONO_GC_END (GENERATION_OLD);
3243         binary_protocol_collection_end (stat_major_gcs - 1, GENERATION_OLD);
3244 }
3245
3246 static gboolean
3247 major_do_collection (const char *reason)
3248 {
3249         TV_DECLARE (all_atv);
3250         TV_DECLARE (all_btv);
3251         int old_next_pin_slot;
3252
3253         if (major_collector.get_and_reset_num_major_objects_marked) {
3254                 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
3255                 g_assert (!num_marked);
3256         }
3257
3258         /* world must be stopped already */
3259         TV_GETTIME (all_atv);
3260
3261         major_start_collection (FALSE, &old_next_pin_slot);
3262         major_finish_collection (reason, old_next_pin_slot, FALSE);
3263
3264         TV_GETTIME (all_btv);
3265         gc_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3266
3267         /* FIXME: also report this to the user, preferably in gc-end. */
3268         if (major_collector.get_and_reset_num_major_objects_marked)
3269                 major_collector.get_and_reset_num_major_objects_marked ();
3270
3271         return bytes_pinned_from_failed_allocation > 0;
3272 }
3273
3274 static gboolean major_do_collection (const char *reason);
3275
3276 static void
3277 major_start_concurrent_collection (const char *reason)
3278 {
3279         long long num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3280
3281         g_assert (num_objects_marked == 0);
3282
3283         MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
3284
3285         // FIXME: store reason and pass it when finishing
3286         major_start_collection (TRUE, NULL);
3287
3288         gray_queue_redirect (&gray_queue);
3289         sgen_workers_wait_for_jobs ();
3290
3291         num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3292         MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
3293
3294         current_collection_generation = -1;
3295 }
3296
3297 static gboolean
3298 major_update_or_finish_concurrent_collection (gboolean force_finish)
3299 {
3300         SgenGrayQueue unpin_queue;
3301         memset (&unpin_queue, 0, sizeof (unpin_queue));
3302
3303         MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3304
3305         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3306
3307         major_collector.update_cardtable_mod_union ();
3308         sgen_los_update_cardtable_mod_union ();
3309
3310         if (!force_finish && !sgen_workers_all_done ()) {
3311                 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3312                 return FALSE;
3313         }
3314
3315         collect_nursery (&unpin_queue, TRUE);
3316
3317         current_collection_generation = GENERATION_OLD;
3318         major_finish_collection ("finishing", -1, TRUE);
3319
3320         if (whole_heap_check_before_collection)
3321                 sgen_check_whole_heap (FALSE);
3322
3323         unpin_objects_from_queue (&unpin_queue);
3324         sgen_gray_object_queue_deinit (&unpin_queue);
3325
3326         MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3327
3328         current_collection_generation = -1;
3329
3330         return TRUE;
3331 }
3332
3333 /*
3334  * Ensure an allocation request for @size will succeed by freeing enough memory.
3335  *
3336  * LOCKING: The GC lock MUST be held.
3337  */
3338 void
3339 sgen_ensure_free_space (size_t size)
3340 {
3341         int generation_to_collect = -1;
3342         const char *reason = NULL;
3343
3344
3345         if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3346                 if (sgen_need_major_collection (size)) {
3347                         reason = "LOS overflow";
3348                         generation_to_collect = GENERATION_OLD;
3349                 }
3350         } else {
3351                 if (degraded_mode) {
3352                         if (sgen_need_major_collection (size)) {
3353                                 reason = "Degraded mode overflow";
3354                                 generation_to_collect = GENERATION_OLD;
3355                         }
3356                 } else if (sgen_need_major_collection (size)) {
3357                         reason = "Minor allowance";
3358                         generation_to_collect = GENERATION_OLD;
3359                 } else {
3360                         generation_to_collect = GENERATION_NURSERY;
3361                         reason = "Nursery full";
3362                 }
3363         }
3364
3365         if (generation_to_collect == -1) {
3366                 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3367                         generation_to_collect = GENERATION_OLD;
3368                         reason = "Finish concurrent collection";
3369                 }
3370         }
3371
3372         if (generation_to_collect == -1)
3373                 return;
3374         sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3375 }
3376
3377 void
3378 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3379 {
3380         TV_DECLARE (gc_end);
3381         GGTimingInfo infos [2];
3382         int overflow_generation_to_collect = -1;
3383         int oldest_generation_collected = generation_to_collect;
3384         const char *overflow_reason = NULL;
3385
3386         MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3387
3388         g_assert (generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD);
3389
3390         memset (infos, 0, sizeof (infos));
3391         mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3392
3393         infos [0].generation = generation_to_collect;
3394         infos [0].reason = reason;
3395         infos [0].is_overflow = FALSE;
3396         TV_GETTIME (infos [0].total_time);
3397         infos [1].generation = -1;
3398
3399         sgen_stop_world (generation_to_collect);
3400
3401         if (concurrent_collection_in_progress) {
3402                 if (major_update_or_finish_concurrent_collection (wait_to_finish && generation_to_collect == GENERATION_OLD)) {
3403                         oldest_generation_collected = GENERATION_OLD;
3404                         goto done;
3405                 }
3406                 if (generation_to_collect == GENERATION_OLD)
3407                         goto done;
3408         }
3409
3410         //FIXME extract overflow reason
3411         if (generation_to_collect == GENERATION_NURSERY) {
3412                 if (collect_nursery (NULL, FALSE)) {
3413                         overflow_generation_to_collect = GENERATION_OLD;
3414                         overflow_reason = "Minor overflow";
3415                 }
3416         } else {
3417                 if (major_collector.is_concurrent) {
3418                         g_assert (!concurrent_collection_in_progress);
3419                         if (!wait_to_finish)
3420                                 collect_nursery (NULL, FALSE);
3421                 }
3422
3423                 if (major_collector.is_concurrent && !wait_to_finish) {
3424                         major_start_concurrent_collection (reason);
3425                         // FIXME: set infos[0] properly
3426                         goto done;
3427                 } else {
3428                         if (major_do_collection (reason)) {
3429                                 overflow_generation_to_collect = GENERATION_NURSERY;
3430                                 overflow_reason = "Excessive pinning";
3431                         }
3432                 }
3433         }
3434
3435         TV_GETTIME (gc_end);
3436         infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end);
3437
3438
3439         if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) {
3440                 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3441                 infos [1].generation = overflow_generation_to_collect;
3442                 infos [1].reason = overflow_reason;
3443                 infos [1].is_overflow = TRUE;
3444                 infos [1].total_time = gc_end;
3445
3446                 if (overflow_generation_to_collect == GENERATION_NURSERY)
3447                         collect_nursery (NULL, FALSE);
3448                 else
3449                         major_do_collection (overflow_reason);
3450
3451                 TV_GETTIME (gc_end);
3452                 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3453
3454                 /* keep events symmetric */
3455                 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3456
3457                 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3458         }
3459
3460         SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3461
3462         /* this also sets the proper pointers for the next allocation */
3463         if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3464                 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3465                 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%d pinned)", requested_size, sgen_get_pinned_count ());
3466                 sgen_dump_pin_queue ();
3467                 degraded_mode = 1;
3468         }
3469
3470  done:
3471         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3472
3473         sgen_restart_world (oldest_generation_collected, infos);
3474
3475         mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3476 }
3477
3478 /*
3479  * ######################################################################
3480  * ########  Memory allocation from the OS
3481  * ######################################################################
3482  * This section of code deals with getting memory from the OS and
3483  * allocating memory for GC-internal data structures.
3484  * Internal memory can be handled with a freelist for small objects.
3485  */
3486
3487 /*
3488  * Debug reporting.
3489  */
3490 G_GNUC_UNUSED static void
3491 report_internal_mem_usage (void)
3492 {
3493         printf ("Internal memory usage:\n");
3494         sgen_report_internal_mem_usage ();
3495         printf ("Pinned memory usage:\n");
3496         major_collector.report_pinned_memory_usage ();
3497 }
3498
3499 /*
3500  * ######################################################################
3501  * ########  Finalization support
3502  * ######################################################################
3503  */
3504
3505 static inline gboolean
3506 sgen_major_is_object_alive (void *object)
3507 {
3508         mword objsize;
3509
3510         /* Oldgen objects can be pinned and forwarded too */
3511         if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3512                 return TRUE;
3513
3514         /*
3515          * FIXME: major_collector.is_object_live() also calculates the
3516          * size.  Avoid the double calculation.
3517          */
3518         objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3519         if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3520                 return sgen_los_object_is_pinned (object);
3521
3522         return major_collector.is_object_live (object);
3523 }
3524
3525 /*
3526  * If the object has been forwarded it means it's still referenced from a root.
3527  * If it is pinned it's still alive as well.
3528  * A LOS object is only alive if we have pinned it.
3529  * Return TRUE if @obj is ready to be finalized.
3530  */
3531 static inline gboolean
3532 sgen_is_object_alive (void *object)
3533 {
3534         if (ptr_in_nursery (object))
3535                 return sgen_nursery_is_object_alive (object);
3536
3537         return sgen_major_is_object_alive (object);
3538 }
3539
3540 /*
3541  * This function returns true if @object is either alive or it belongs to the old gen
3542  * and we're currently doing a minor collection.
3543  */
3544 static inline int
3545 sgen_is_object_alive_for_current_gen (char *object)
3546 {
3547         if (ptr_in_nursery (object))
3548                 return sgen_nursery_is_object_alive (object);
3549
3550         if (current_collection_generation == GENERATION_NURSERY)
3551                 return TRUE;
3552
3553         return sgen_major_is_object_alive (object);
3554 }
3555
3556 /*
3557  * This function returns true if @object is either alive and belongs to the
3558  * current collection - major collections are full heap, so old gen objects
3559  * are never alive during a minor collection.
3560  */
3561 static inline int
3562 sgen_is_object_alive_and_on_current_collection (char *object)
3563 {
3564         if (ptr_in_nursery (object))
3565                 return sgen_nursery_is_object_alive (object);
3566
3567         if (current_collection_generation == GENERATION_NURSERY)
3568                 return FALSE;
3569
3570         return sgen_major_is_object_alive (object);
3571 }
3572
3573
3574 gboolean
3575 sgen_gc_is_object_ready_for_finalization (void *object)
3576 {
3577         return !sgen_is_object_alive (object);
3578 }
3579
3580 static gboolean
3581 has_critical_finalizer (MonoObject *obj)
3582 {
3583         MonoClass *class;
3584
3585         if (!mono_defaults.critical_finalizer_object)
3586                 return FALSE;
3587
3588         class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3589
3590         return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3591 }
3592
3593 void
3594 sgen_queue_finalization_entry (MonoObject *obj)
3595 {
3596         FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3597         gboolean critical = has_critical_finalizer (obj);
3598         entry->object = obj;
3599         if (critical) {
3600                 entry->next = critical_fin_list;
3601                 critical_fin_list = entry;
3602         } else {
3603                 entry->next = fin_ready_list;
3604                 fin_ready_list = entry;
3605         }
3606
3607 #ifdef ENABLE_DTRACE
3608         if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3609                 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3610                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3611                 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3612                                 vt->klass->name_space, vt->klass->name, gen, critical);
3613         }
3614 #endif
3615 }
3616
3617 gboolean
3618 sgen_object_is_live (void *obj)
3619 {
3620         return sgen_is_object_alive_and_on_current_collection (obj);
3621 }
3622
3623 /* LOCKING: requires that the GC lock is held */
3624 static void
3625 null_ephemerons_for_domain (MonoDomain *domain)
3626 {
3627         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3628
3629         while (current) {
3630                 MonoObject *object = (MonoObject*)current->array;
3631
3632                 if (object && !object->vtable) {
3633                         EphemeronLinkNode *tmp = current;
3634
3635                         if (prev)
3636                                 prev->next = current->next;
3637                         else
3638                                 ephemeron_list = current->next;
3639
3640                         current = current->next;
3641                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3642                 } else {
3643                         prev = current;
3644                         current = current->next;
3645                 }
3646         }
3647 }
3648
3649 /* LOCKING: requires that the GC lock is held */
3650 static void
3651 clear_unreachable_ephemerons (ScanCopyContext ctx)
3652 {
3653         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3654         GrayQueue *queue = ctx.queue;
3655         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3656         MonoArray *array;
3657         Ephemeron *cur, *array_end;
3658         char *tombstone;
3659
3660         while (current) {
3661                 char *object = current->array;
3662
3663                 if (!sgen_is_object_alive_for_current_gen (object)) {
3664                         EphemeronLinkNode *tmp = current;
3665
3666                         SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3667
3668                         if (prev)
3669                                 prev->next = current->next;
3670                         else
3671                                 ephemeron_list = current->next;
3672
3673                         current = current->next;
3674                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3675
3676                         continue;
3677                 }
3678
3679                 copy_func ((void**)&object, queue);
3680                 current->array = object;
3681
3682                 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3683
3684                 array = (MonoArray*)object;
3685                 cur = mono_array_addr (array, Ephemeron, 0);
3686                 array_end = cur + mono_array_length_fast (array);
3687                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3688
3689                 for (; cur < array_end; ++cur) {
3690                         char *key = (char*)cur->key;
3691
3692                         if (!key || key == tombstone)
3693                                 continue;
3694
3695                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3696                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3697                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3698
3699                         if (!sgen_is_object_alive_for_current_gen (key)) {
3700                                 cur->key = tombstone;
3701                                 cur->value = NULL;
3702                                 continue;
3703                         }
3704                 }
3705                 prev = current;
3706                 current = current->next;
3707         }
3708 }
3709
3710 /*
3711 LOCKING: requires that the GC lock is held
3712
3713 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3714 */
3715 static int
3716 mark_ephemerons_in_range (ScanCopyContext ctx)
3717 {
3718         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3719         GrayQueue *queue = ctx.queue;
3720         int nothing_marked = 1;
3721         EphemeronLinkNode *current = ephemeron_list;
3722         MonoArray *array;
3723         Ephemeron *cur, *array_end;
3724         char *tombstone;
3725
3726         for (current = ephemeron_list; current; current = current->next) {
3727                 char *object = current->array;
3728                 SGEN_LOG (5, "Ephemeron array at %p", object);
3729
3730                 /*It has to be alive*/
3731                 if (!sgen_is_object_alive_for_current_gen (object)) {
3732                         SGEN_LOG (5, "\tnot reachable");
3733                         continue;
3734                 }
3735
3736                 copy_func ((void**)&object, queue);
3737
3738                 array = (MonoArray*)object;
3739                 cur = mono_array_addr (array, Ephemeron, 0);
3740                 array_end = cur + mono_array_length_fast (array);
3741                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3742
3743                 for (; cur < array_end; ++cur) {
3744                         char *key = cur->key;
3745
3746                         if (!key || key == tombstone)
3747                                 continue;
3748
3749                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3750                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3751                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3752
3753                         if (sgen_is_object_alive_for_current_gen (key)) {
3754                                 char *value = cur->value;
3755
3756                                 copy_func ((void**)&cur->key, queue);
3757                                 if (value) {
3758                                         if (!sgen_is_object_alive_for_current_gen (value))
3759                                                 nothing_marked = 0;
3760                                         copy_func ((void**)&cur->value, queue);
3761                                 }
3762                         }
3763                 }
3764         }
3765
3766         SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3767         return nothing_marked;
3768 }
3769
3770 int
3771 mono_gc_invoke_finalizers (void)
3772 {
3773         FinalizeReadyEntry *entry = NULL;
3774         gboolean entry_is_critical = FALSE;
3775         int count = 0;
3776         void *obj;
3777         /* FIXME: batch to reduce lock contention */
3778         while (fin_ready_list || critical_fin_list) {
3779                 LOCK_GC;
3780
3781                 if (entry) {
3782                         FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3783
3784                         /* We have finalized entry in the last
3785                            interation, now we need to remove it from
3786                            the list. */
3787                         if (*list == entry)
3788                                 *list = entry->next;
3789                         else {
3790                                 FinalizeReadyEntry *e = *list;
3791                                 while (e->next != entry)
3792                                         e = e->next;
3793                                 e->next = entry->next;
3794                         }
3795                         sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3796                         entry = NULL;
3797                 }
3798
3799                 /* Now look for the first non-null entry. */
3800                 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3801                         ;
3802                 if (entry) {
3803                         entry_is_critical = FALSE;
3804                 } else {
3805                         entry_is_critical = TRUE;
3806                         for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3807                                 ;
3808                 }
3809
3810                 if (entry) {
3811                         g_assert (entry->object);
3812                         num_ready_finalizers--;
3813                         obj = entry->object;
3814                         entry->object = NULL;
3815                         SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3816                 }
3817
3818                 UNLOCK_GC;
3819
3820                 if (!entry)
3821                         break;
3822
3823                 g_assert (entry->object == NULL);
3824                 count++;
3825                 /* the object is on the stack so it is pinned */
3826                 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3827                 mono_gc_run_finalize (obj, NULL);
3828         }
3829         g_assert (!entry);
3830         return count;
3831 }
3832
3833 gboolean
3834 mono_gc_pending_finalizers (void)
3835 {
3836         return fin_ready_list || critical_fin_list;
3837 }
3838
3839 /*
3840  * ######################################################################
3841  * ########  registered roots support
3842  * ######################################################################
3843  */
3844
3845 /*
3846  * We do not coalesce roots.
3847  */
3848 static int
3849 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3850 {
3851         RootRecord new_root;
3852         int i;
3853         LOCK_GC;
3854         for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3855                 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3856                 /* we allow changing the size and the descriptor (for thread statics etc) */
3857                 if (root) {
3858                         size_t old_size = root->end_root - start;
3859                         root->end_root = start + size;
3860                         g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3861                                           ((root->root_desc == 0) && (descr == NULL)));
3862                         root->root_desc = (mword)descr;
3863                         roots_size += size;
3864                         roots_size -= old_size;
3865                         UNLOCK_GC;
3866                         return TRUE;
3867                 }
3868         }
3869
3870         new_root.end_root = start + size;
3871         new_root.root_desc = (mword)descr;
3872
3873         sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3874         roots_size += size;
3875
3876         SGEN_LOG (3, "Added root for range: %p-%p, descr: %p  (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3877
3878         UNLOCK_GC;
3879         return TRUE;
3880 }
3881
3882 int
3883 mono_gc_register_root (char *start, size_t size, void *descr)
3884 {
3885         return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3886 }
3887
3888 int
3889 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3890 {
3891         return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3892 }
3893
3894 void
3895 mono_gc_deregister_root (char* addr)
3896 {
3897         int root_type;
3898         RootRecord root;
3899
3900         LOCK_GC;
3901         for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3902                 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3903                         roots_size -= (root.end_root - addr);
3904         }
3905         UNLOCK_GC;
3906 }
3907
3908 /*
3909  * ######################################################################
3910  * ########  Thread handling (stop/start code)
3911  * ######################################################################
3912  */
3913
3914 unsigned int sgen_global_stop_count = 0;
3915
3916 void
3917 sgen_fill_thread_info_for_suspend (SgenThreadInfo *info)
3918 {
3919         if (remset.fill_thread_info_for_suspend)
3920                 remset.fill_thread_info_for_suspend (info);
3921 }
3922
3923 int
3924 sgen_get_current_collection_generation (void)
3925 {
3926         return current_collection_generation;
3927 }
3928
3929 void
3930 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3931 {
3932         gc_callbacks = *callbacks;
3933 }
3934
3935 MonoGCCallbacks *
3936 mono_gc_get_gc_callbacks ()
3937 {
3938         return &gc_callbacks;
3939 }
3940
3941 /* Variables holding start/end nursery so it won't have to be passed at every call */
3942 static void *scan_area_arg_start, *scan_area_arg_end;
3943
3944 void
3945 mono_gc_conservatively_scan_area (void *start, void *end)
3946 {
3947         conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
3948 }
3949
3950 void*
3951 mono_gc_scan_object (void *obj)
3952 {
3953         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
3954         current_object_ops.copy_or_mark_object (&obj, data->queue);
3955         return obj;
3956 }
3957
3958 /*
3959  * Mark from thread stacks and registers.
3960  */
3961 static void
3962 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
3963 {
3964         SgenThreadInfo *info;
3965
3966         scan_area_arg_start = start_nursery;
3967         scan_area_arg_end = end_nursery;
3968
3969         FOREACH_THREAD (info) {
3970                 if (info->skip) {
3971                         SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3972                         continue;
3973                 }
3974                 if (info->gc_disabled) {
3975                         SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3976                         continue;
3977                 }
3978
3979                 if (!info->joined_stw) {
3980                         SGEN_LOG (3, "Skipping thread not seen in STW %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3981                         continue;
3982                 }
3983
3984                 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
3985                 if (!info->thread_is_dying) {
3986                         if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
3987                                 UserCopyOrMarkData data = { NULL, queue };
3988                                 set_user_copy_or_mark_data (&data);
3989                                 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
3990                                 set_user_copy_or_mark_data (NULL);
3991                         } else if (!precise) {
3992                                 if (!conservative_stack_mark) {
3993                                         fprintf (stderr, "Precise stack mark not supported - disabling.\n");
3994                                         conservative_stack_mark = TRUE;
3995                                 }
3996                                 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
3997                         }
3998                 }
3999
4000                 if (!info->thread_is_dying && !precise) {
4001 #ifdef USE_MONO_CTX
4002                         conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
4003                                 start_nursery, end_nursery, PIN_TYPE_STACK);
4004 #else
4005                         conservatively_pin_objects_from (&info->regs, &info->regs + ARCH_NUM_REGS,
4006                                         start_nursery, end_nursery, PIN_TYPE_STACK);
4007 #endif
4008                 }
4009         } END_FOREACH_THREAD
4010 }
4011
4012 static gboolean
4013 ptr_on_stack (void *ptr)
4014 {
4015         gpointer stack_start = &stack_start;
4016         SgenThreadInfo *info = mono_thread_info_current ();
4017
4018         if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
4019                 return TRUE;
4020         return FALSE;
4021 }
4022
4023 static void*
4024 sgen_thread_register (SgenThreadInfo* info, void *addr)
4025 {
4026 #ifndef HAVE_KW_THREAD
4027         SgenThreadInfo *__thread_info__ = info;
4028 #endif
4029
4030         LOCK_GC;
4031 #ifndef HAVE_KW_THREAD
4032         info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
4033
4034         g_assert (!mono_native_tls_get_value (thread_info_key));
4035         mono_native_tls_set_value (thread_info_key, info);
4036 #else
4037         sgen_thread_info = info;
4038 #endif
4039
4040 #if !defined(__MACH__)
4041         info->stop_count = -1;
4042         info->signal = 0;
4043 #endif
4044         info->skip = 0;
4045         info->joined_stw = FALSE;
4046         info->doing_handshake = FALSE;
4047         info->thread_is_dying = FALSE;
4048         info->stack_start = NULL;
4049         info->store_remset_buffer_addr = &STORE_REMSET_BUFFER;
4050         info->store_remset_buffer_index_addr = &STORE_REMSET_BUFFER_INDEX;
4051         info->stopped_ip = NULL;
4052         info->stopped_domain = NULL;
4053 #ifdef USE_MONO_CTX
4054         memset (&info->ctx, 0, sizeof (MonoContext));
4055 #else
4056         memset (&info->regs, 0, sizeof (info->regs));
4057 #endif
4058
4059         sgen_init_tlab_info (info);
4060
4061         binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
4062
4063 #ifdef HAVE_KW_THREAD
4064         store_remset_buffer_index_addr = &store_remset_buffer_index;
4065 #endif
4066
4067         /* try to get it with attributes first */
4068 #if (defined(HAVE_PTHREAD_GETATTR_NP) || defined(HAVE_PTHREAD_ATTR_GET_NP)) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
4069   {
4070      size_t size;
4071      void *sstart;
4072      pthread_attr_t attr;
4073
4074 #if defined(HAVE_PTHREAD_GETATTR_NP)
4075     /* Linux */
4076     pthread_getattr_np (pthread_self (), &attr);
4077 #elif defined(HAVE_PTHREAD_ATTR_GET_NP)
4078     /* BSD */
4079     pthread_attr_init (&attr);
4080     pthread_attr_get_np (pthread_self (), &attr);
4081 #else
4082 #error Cannot determine which API is needed to retrieve pthread attributes.
4083 #endif
4084
4085      pthread_attr_getstack (&attr, &sstart, &size);
4086      info->stack_start_limit = sstart;
4087      info->stack_end = (char*)sstart + size;
4088      pthread_attr_destroy (&attr);
4089   }
4090 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
4091                  info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
4092                  info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
4093 #else
4094         {
4095                 /* FIXME: we assume the stack grows down */
4096                 gsize stack_bottom = (gsize)addr;
4097                 stack_bottom += 4095;
4098                 stack_bottom &= ~4095;
4099                 info->stack_end = (char*)stack_bottom;
4100         }
4101 #endif
4102
4103 #ifdef HAVE_KW_THREAD
4104         stack_end = info->stack_end;
4105 #endif
4106
4107         if (remset.register_thread)
4108                 remset.register_thread (info);
4109
4110         SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
4111
4112         if (gc_callbacks.thread_attach_func)
4113                 info->runtime_data = gc_callbacks.thread_attach_func ();
4114
4115         UNLOCK_GC;
4116         return info;
4117 }
4118
4119 static void
4120 sgen_wbarrier_cleanup_thread (SgenThreadInfo *p)
4121 {
4122         if (remset.cleanup_thread)
4123                 remset.cleanup_thread (p);
4124 }
4125
4126 static void
4127 sgen_thread_unregister (SgenThreadInfo *p)
4128 {
4129         /* If a delegate is passed to native code and invoked on a thread we dont
4130          * know about, the jit will register it with mono_jit_thread_attach, but
4131          * we have no way of knowing when that thread goes away.  SGen has a TSD
4132          * so we assume that if the domain is still registered, we can detach
4133          * the thread
4134          */
4135         if (mono_domain_get ())
4136                 mono_thread_detach (mono_thread_current ());
4137
4138         p->thread_is_dying = TRUE;
4139
4140         /*
4141         There is a race condition between a thread finishing executing and been removed
4142         from the GC thread set.
4143         This happens on posix systems when TLS data is been cleaned-up, libpthread will
4144         set the thread_info slot to NULL before calling the cleanup function. This
4145         opens a window in which the thread is registered but has a NULL TLS.
4146
4147         The suspend signal handler needs TLS data to know where to store thread state
4148         data or otherwise it will simply ignore the thread.
4149
4150         This solution works because the thread doing STW will wait until all threads been
4151         suspended handshake back, so there is no race between the doing_hankshake test
4152         and the suspend_thread call.
4153
4154         This is not required on systems that do synchronous STW as those can deal with
4155         the above race at suspend time.
4156
4157         FIXME: I believe we could avoid this by using mono_thread_info_lookup when
4158         mono_thread_info_current returns NULL. Or fix mono_thread_info_lookup to do so.
4159         */
4160 #if (defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED) || !defined(HAVE_PTHREAD_KILL)
4161         LOCK_GC;
4162 #else
4163         while (!TRYLOCK_GC) {
4164                 if (!sgen_park_current_thread_if_doing_handshake (p))
4165                         g_usleep (50);
4166         }
4167         MONO_GC_LOCKED ();
4168 #endif
4169
4170         binary_protocol_thread_unregister ((gpointer)mono_thread_info_get_tid (p));
4171         SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)mono_thread_info_get_tid (p));
4172
4173         if (gc_callbacks.thread_detach_func) {
4174                 gc_callbacks.thread_detach_func (p->runtime_data);
4175                 p->runtime_data = NULL;
4176         }
4177         sgen_wbarrier_cleanup_thread (p);
4178
4179         mono_threads_unregister_current_thread (p);
4180         UNLOCK_GC;
4181 }
4182
4183
4184 static void
4185 sgen_thread_attach (SgenThreadInfo *info)
4186 {
4187         LOCK_GC;
4188         /*this is odd, can we get attached before the gc is inited?*/
4189         init_stats ();
4190         UNLOCK_GC;
4191
4192         if (gc_callbacks.thread_attach_func && !info->runtime_data)
4193                 info->runtime_data = gc_callbacks.thread_attach_func ();
4194 }
4195 gboolean
4196 mono_gc_register_thread (void *baseptr)
4197 {
4198         return mono_thread_info_attach (baseptr) != NULL;
4199 }
4200
4201 /*
4202  * mono_gc_set_stack_end:
4203  *
4204  *   Set the end of the current threads stack to STACK_END. The stack space between
4205  * STACK_END and the real end of the threads stack will not be scanned during collections.
4206  */
4207 void
4208 mono_gc_set_stack_end (void *stack_end)
4209 {
4210         SgenThreadInfo *info;
4211
4212         LOCK_GC;
4213         info = mono_thread_info_current ();
4214         if (info) {
4215                 g_assert (stack_end < info->stack_end);
4216                 info->stack_end = stack_end;
4217         }
4218         UNLOCK_GC;
4219 }
4220
4221 #if USE_PTHREAD_INTERCEPT
4222
4223
4224 int
4225 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
4226 {
4227         return pthread_create (new_thread, attr, start_routine, arg);
4228 }
4229
4230 int
4231 mono_gc_pthread_join (pthread_t thread, void **retval)
4232 {
4233         return pthread_join (thread, retval);
4234 }
4235
4236 int
4237 mono_gc_pthread_detach (pthread_t thread)
4238 {
4239         return pthread_detach (thread);
4240 }
4241
4242 void
4243 mono_gc_pthread_exit (void *retval)
4244 {
4245         pthread_exit (retval);
4246 }
4247
4248 #endif /* USE_PTHREAD_INTERCEPT */
4249
4250 /*
4251  * ######################################################################
4252  * ########  Write barriers
4253  * ######################################################################
4254  */
4255
4256 /*
4257  * Note: the write barriers first do the needed GC work and then do the actual store:
4258  * this way the value is visible to the conservative GC scan after the write barrier
4259  * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
4260  * the conservative scan, otherwise by the remembered set scan.
4261  */
4262 void
4263 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
4264 {
4265         HEAVY_STAT (++stat_wbarrier_set_field);
4266         if (ptr_in_nursery (field_ptr)) {
4267                 *(void**)field_ptr = value;
4268                 return;
4269         }
4270         SGEN_LOG (8, "Adding remset at %p", field_ptr);
4271         if (value)
4272                 binary_protocol_wbarrier (field_ptr, value, value->vtable);
4273
4274         remset.wbarrier_set_field (obj, field_ptr, value);
4275 }
4276
4277 void
4278 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
4279 {
4280         HEAVY_STAT (++stat_wbarrier_set_arrayref);
4281         if (ptr_in_nursery (slot_ptr)) {
4282                 *(void**)slot_ptr = value;
4283                 return;
4284         }
4285         SGEN_LOG (8, "Adding remset at %p", slot_ptr);
4286         if (value)
4287                 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
4288
4289         remset.wbarrier_set_arrayref (arr, slot_ptr, value);
4290 }
4291
4292 void
4293 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
4294 {
4295         HEAVY_STAT (++stat_wbarrier_arrayref_copy);
4296         /*This check can be done without taking a lock since dest_ptr array is pinned*/
4297         if (ptr_in_nursery (dest_ptr) || count <= 0) {
4298                 mono_gc_memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
4299                 return;
4300         }
4301
4302 #ifdef SGEN_BINARY_PROTOCOL
4303         {
4304                 int i;
4305                 for (i = 0; i < count; ++i) {
4306                         gpointer dest = (gpointer*)dest_ptr + i;
4307                         gpointer obj = *((gpointer*)src_ptr + i);
4308                         if (obj)
4309                                 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
4310                 }
4311         }
4312 #endif
4313
4314         remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
4315 }
4316
4317 static char *found_obj;
4318
4319 static void
4320 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4321 {
4322         char *ptr = user_data;
4323
4324         if (ptr >= obj && ptr < obj + size) {
4325                 g_assert (!found_obj);
4326                 found_obj = obj;
4327         }
4328 }
4329
4330 /* for use in the debugger */
4331 char* find_object_for_ptr (char *ptr);
4332 char*
4333 find_object_for_ptr (char *ptr)
4334 {
4335         if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4336                 found_obj = NULL;
4337                 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4338                                 find_object_for_ptr_callback, ptr, TRUE);
4339                 if (found_obj)
4340                         return found_obj;
4341         }
4342
4343         found_obj = NULL;
4344         sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4345         if (found_obj)
4346                 return found_obj;
4347
4348         /*
4349          * Very inefficient, but this is debugging code, supposed to
4350          * be called from gdb, so we don't care.
4351          */
4352         found_obj = NULL;
4353         major_collector.iterate_objects (TRUE, TRUE, find_object_for_ptr_callback, ptr);
4354         return found_obj;
4355 }
4356
4357 void
4358 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4359 {
4360         gpointer obj;
4361
4362         HEAVY_STAT (++stat_wbarrier_generic_store);
4363
4364 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4365         /* FIXME: ptr_in_heap must be called with the GC lock held */
4366         if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4367                 char *start = find_object_for_ptr (ptr);
4368                 MonoObject *value = *(MonoObject**)ptr;
4369                 LOCK_GC;
4370                 g_assert (start);
4371                 if (start) {
4372                         MonoObject *obj = (MonoObject*)start;
4373                         if (obj->vtable->domain != value->vtable->domain)
4374                                 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4375                 }
4376                 UNLOCK_GC;
4377         }
4378 #endif
4379
4380         obj = *(gpointer*)ptr;
4381         if (obj)
4382                 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4383
4384         if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4385                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4386                 return;
4387         }
4388
4389         /*
4390          * We need to record old->old pointer locations for the
4391          * concurrent collector.
4392          */
4393         if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4394                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4395                 return;
4396         }
4397
4398         SGEN_LOG (8, "Adding remset at %p", ptr);
4399
4400         remset.wbarrier_generic_nostore (ptr);
4401 }
4402
4403 void
4404 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4405 {
4406         SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4407         *(void**)ptr = value;
4408         if (ptr_in_nursery (value))
4409                 mono_gc_wbarrier_generic_nostore (ptr);
4410         sgen_dummy_use (value);
4411 }
4412
4413 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4414 {
4415         mword *dest = _dest;
4416         mword *src = _src;
4417
4418         while (size) {
4419                 if (bitmap & 0x1)
4420                         mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4421                 else
4422                         *dest = *src;
4423                 ++src;
4424                 ++dest;
4425                 size -= SIZEOF_VOID_P;
4426                 bitmap >>= 1;
4427         }
4428 }
4429
4430 #ifdef SGEN_BINARY_PROTOCOL
4431 #undef HANDLE_PTR
4432 #define HANDLE_PTR(ptr,obj) do {                                        \
4433                 gpointer o = *(gpointer*)(ptr);                         \
4434                 if ((o)) {                                              \
4435                         gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4436                         binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4437                 }                                                       \
4438         } while (0)
4439
4440 static void
4441 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4442 {
4443 #define SCAN_OBJECT_NOVTABLE
4444 #include "sgen-scan-object.h"
4445 }
4446 #endif
4447
4448 void
4449 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4450 {
4451         HEAVY_STAT (++stat_wbarrier_value_copy);
4452         g_assert (klass->valuetype);
4453
4454         SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4455
4456         if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4457                 size_t element_size = mono_class_value_size (klass, NULL);
4458                 size_t size = count * element_size;
4459                 mono_gc_memmove (dest, src, size);
4460                 return;
4461         }
4462
4463 #ifdef SGEN_BINARY_PROTOCOL
4464         {
4465                 size_t element_size = mono_class_value_size (klass, NULL);
4466                 int i;
4467                 for (i = 0; i < count; ++i) {
4468                         scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4469                                         (char*)src + i * element_size - sizeof (MonoObject),
4470                                         (mword) klass->gc_descr);
4471                 }
4472         }
4473 #endif
4474
4475         remset.wbarrier_value_copy (dest, src, count, klass);
4476 }
4477
4478 /**
4479  * mono_gc_wbarrier_object_copy:
4480  *
4481  * Write barrier to call when obj is the result of a clone or copy of an object.
4482  */
4483 void
4484 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4485 {
4486         int size;
4487
4488         HEAVY_STAT (++stat_wbarrier_object_copy);
4489
4490         if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4491                 size = mono_object_class (obj)->instance_size;
4492                 mono_gc_memmove ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4493                                 size - sizeof (MonoObject));
4494                 return;
4495         }
4496
4497 #ifdef SGEN_BINARY_PROTOCOL
4498         scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4499 #endif
4500
4501         remset.wbarrier_object_copy (obj, src);
4502 }
4503
4504
4505 /*
4506  * ######################################################################
4507  * ########  Other mono public interface functions.
4508  * ######################################################################
4509  */
4510
4511 #define REFS_SIZE 128
4512 typedef struct {
4513         void *data;
4514         MonoGCReferences callback;
4515         int flags;
4516         int count;
4517         int called;
4518         MonoObject *refs [REFS_SIZE];
4519         uintptr_t offsets [REFS_SIZE];
4520 } HeapWalkInfo;
4521
4522 #undef HANDLE_PTR
4523 #define HANDLE_PTR(ptr,obj)     do {    \
4524                 if (*(ptr)) {   \
4525                         if (hwi->count == REFS_SIZE) {  \
4526                                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);    \
4527                                 hwi->count = 0; \
4528                                 hwi->called = 1;        \
4529                         }       \
4530                         hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start;  \
4531                         hwi->refs [hwi->count++] = *(ptr);      \
4532                 }       \
4533         } while (0)
4534
4535 static void
4536 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4537 {
4538 #include "sgen-scan-object.h"
4539 }
4540
4541 static void
4542 walk_references (char *start, size_t size, void *data)
4543 {
4544         HeapWalkInfo *hwi = data;
4545         hwi->called = 0;
4546         hwi->count = 0;
4547         collect_references (hwi, start, size);
4548         if (hwi->count || !hwi->called)
4549                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4550 }
4551
4552 /**
4553  * mono_gc_walk_heap:
4554  * @flags: flags for future use
4555  * @callback: a function pointer called for each object in the heap
4556  * @data: a user data pointer that is passed to callback
4557  *
4558  * This function can be used to iterate over all the live objects in the heap:
4559  * for each object, @callback is invoked, providing info about the object's
4560  * location in memory, its class, its size and the objects it references.
4561  * For each referenced object it's offset from the object address is
4562  * reported in the offsets array.
4563  * The object references may be buffered, so the callback may be invoked
4564  * multiple times for the same object: in all but the first call, the size
4565  * argument will be zero.
4566  * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4567  * profiler event handler.
4568  *
4569  * Returns: a non-zero value if the GC doesn't support heap walking
4570  */
4571 int
4572 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4573 {
4574         HeapWalkInfo hwi;
4575
4576         hwi.flags = flags;
4577         hwi.callback = callback;
4578         hwi.data = data;
4579
4580         sgen_clear_nursery_fragments ();
4581         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4582
4583         major_collector.iterate_objects (TRUE, TRUE, walk_references, &hwi);
4584         sgen_los_iterate_objects (walk_references, &hwi);
4585
4586         return 0;
4587 }
4588
4589 void
4590 mono_gc_collect (int generation)
4591 {
4592         LOCK_GC;
4593         if (generation > 1)
4594                 generation = 1;
4595         sgen_perform_collection (0, generation, "user request", TRUE);
4596         UNLOCK_GC;
4597 }
4598
4599 int
4600 mono_gc_max_generation (void)
4601 {
4602         return 1;
4603 }
4604
4605 int
4606 mono_gc_collection_count (int generation)
4607 {
4608         if (generation == 0)
4609                 return stat_minor_gcs;
4610         return stat_major_gcs;
4611 }
4612
4613 int64_t
4614 mono_gc_get_used_size (void)
4615 {
4616         gint64 tot = 0;
4617         LOCK_GC;
4618         tot = los_memory_usage;
4619         tot += nursery_section->next_data - nursery_section->data;
4620         tot += major_collector.get_used_size ();
4621         /* FIXME: account for pinned objects */
4622         UNLOCK_GC;
4623         return tot;
4624 }
4625
4626 int
4627 mono_gc_get_los_limit (void)
4628 {
4629         return MAX_SMALL_OBJ_SIZE;
4630 }
4631
4632 gboolean
4633 mono_gc_user_markers_supported (void)
4634 {
4635         return TRUE;
4636 }
4637
4638 gboolean
4639 mono_object_is_alive (MonoObject* o)
4640 {
4641         return TRUE;
4642 }
4643
4644 int
4645 mono_gc_get_generation (MonoObject *obj)
4646 {
4647         if (ptr_in_nursery (obj))
4648                 return 0;
4649         return 1;
4650 }
4651
4652 void
4653 mono_gc_enable_events (void)
4654 {
4655 }
4656
4657 void
4658 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4659 {
4660         sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4661 }
4662
4663 void
4664 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4665 {
4666         sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4667 }
4668
4669 MonoObject*
4670 mono_gc_weak_link_get (void **link_addr)
4671 {
4672         void * volatile *link_addr_volatile;
4673         void *ptr;
4674         MonoObject *obj;
4675  retry:
4676         link_addr_volatile = link_addr;
4677         ptr = (void*)*link_addr_volatile;
4678         /*
4679          * At this point we have a hidden pointer.  If the GC runs
4680          * here, it will not recognize the hidden pointer as a
4681          * reference, and if the object behind it is not referenced
4682          * elsewhere, it will be freed.  Once the world is restarted
4683          * we reveal the pointer, giving us a pointer to a freed
4684          * object.  To make sure we don't return it, we load the
4685          * hidden pointer again.  If it's still the same, we can be
4686          * sure the object reference is valid.
4687          */
4688         if (ptr)
4689                 obj = (MonoObject*) REVEAL_POINTER (ptr);
4690         else
4691                 return NULL;
4692
4693         mono_memory_barrier ();
4694
4695         /*
4696          * During the second bridge processing step the world is
4697          * running again.  That step processes all weak links once
4698          * more to null those that refer to dead objects.  Before that
4699          * is completed, those links must not be followed, so we
4700          * conservatively wait for bridge processing when any weak
4701          * link is dereferenced.
4702          */
4703         if (G_UNLIKELY (bridge_processing_in_progress))
4704                 mono_gc_wait_for_bridge_processing ();
4705
4706         if ((void*)*link_addr_volatile != ptr)
4707                 goto retry;
4708
4709         return obj;
4710 }
4711
4712 gboolean
4713 mono_gc_ephemeron_array_add (MonoObject *obj)
4714 {
4715         EphemeronLinkNode *node;
4716
4717         LOCK_GC;
4718
4719         node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4720         if (!node) {
4721                 UNLOCK_GC;
4722                 return FALSE;
4723         }
4724         node->array = (char*)obj;
4725         node->next = ephemeron_list;
4726         ephemeron_list = node;
4727
4728         SGEN_LOG (5, "Registered ephemeron array %p", obj);
4729
4730         UNLOCK_GC;
4731         return TRUE;
4732 }
4733
4734 void*
4735 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4736 {
4737         void *result;
4738         LOCK_INTERRUPTION;
4739         result = func (data);
4740         UNLOCK_INTERRUPTION;
4741         return result;
4742 }
4743
4744 gboolean
4745 mono_gc_is_gc_thread (void)
4746 {
4747         gboolean result;
4748         LOCK_GC;
4749         result = mono_thread_info_current () != NULL;
4750         UNLOCK_GC;
4751         return result;
4752 }
4753
4754 static gboolean
4755 is_critical_method (MonoMethod *method)
4756 {
4757         return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4758 }
4759
4760 void
4761 mono_gc_base_init (void)
4762 {
4763         MonoThreadInfoCallbacks cb;
4764         char *env;
4765         char **opts, **ptr;
4766         char *major_collector_opt = NULL;
4767         char *minor_collector_opt = NULL;
4768         glong max_heap = 0;
4769         glong soft_limit = 0;
4770         int num_workers;
4771         int result;
4772         int dummy;
4773         gboolean debug_print_allowance = FALSE;
4774         double allowance_ratio = 0, save_target = 0;
4775         gboolean have_split_nursery = FALSE;
4776         gboolean cement_enabled = TRUE;
4777
4778         do {
4779                 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4780                 switch (result) {
4781                 case 1:
4782                         /* already inited */
4783                         return;
4784                 case -1:
4785                         /* being inited by another thread */
4786                         g_usleep (1000);
4787                         break;
4788                 case 0:
4789                         /* we will init it */
4790                         break;
4791                 default:
4792                         g_assert_not_reached ();
4793                 }
4794         } while (result != 0);
4795
4796         LOCK_INIT (gc_mutex);
4797
4798         pagesize = mono_pagesize ();
4799         gc_debug_file = stderr;
4800
4801         cb.thread_register = sgen_thread_register;
4802         cb.thread_unregister = sgen_thread_unregister;
4803         cb.thread_attach = sgen_thread_attach;
4804         cb.mono_method_is_critical = (gpointer)is_critical_method;
4805 #ifndef HOST_WIN32
4806         cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4807 #endif
4808
4809         mono_threads_init (&cb, sizeof (SgenThreadInfo));
4810
4811         LOCK_INIT (sgen_interruption_mutex);
4812         LOCK_INIT (pin_queue_mutex);
4813
4814         init_user_copy_or_mark_key ();
4815
4816         if ((env = getenv ("MONO_GC_PARAMS"))) {
4817                 opts = g_strsplit (env, ",", -1);
4818                 for (ptr = opts; *ptr; ++ptr) {
4819                         char *opt = *ptr;
4820                         if (g_str_has_prefix (opt, "major=")) {
4821                                 opt = strchr (opt, '=') + 1;
4822                                 major_collector_opt = g_strdup (opt);
4823                         } else if (g_str_has_prefix (opt, "minor=")) {
4824                                 opt = strchr (opt, '=') + 1;
4825                                 minor_collector_opt = g_strdup (opt);
4826                         }
4827                 }
4828         } else {
4829                 opts = NULL;
4830         }
4831
4832         init_stats ();
4833         sgen_init_internal_allocator ();
4834         sgen_init_nursery_allocator ();
4835
4836         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4837         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4838         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4839         g_assert (sizeof (GenericStoreRememberedSet) == sizeof (gpointer) * STORE_REMSET_BUFFER_SIZE);
4840         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_STORE_REMSET, sizeof (GenericStoreRememberedSet));
4841         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4842
4843 #ifndef HAVE_KW_THREAD
4844         mono_native_tls_alloc (&thread_info_key, NULL);
4845 #endif
4846
4847         /*
4848          * This needs to happen before any internal allocations because
4849          * it inits the small id which is required for hazard pointer
4850          * operations.
4851          */
4852         sgen_os_init ();
4853
4854         mono_thread_info_attach (&dummy);
4855
4856         if (!minor_collector_opt) {
4857                 sgen_simple_nursery_init (&sgen_minor_collector);
4858         } else {
4859                 if (!strcmp (minor_collector_opt, "simple")) {
4860                         sgen_simple_nursery_init (&sgen_minor_collector);
4861                 } else if (!strcmp (minor_collector_opt, "split")) {
4862                         sgen_split_nursery_init (&sgen_minor_collector);
4863                         have_split_nursery = TRUE;
4864                 } else {
4865                         fprintf (stderr, "Unknown minor collector `%s'.\n", minor_collector_opt);
4866                         exit (1);
4867                 }
4868         }
4869
4870         if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4871                 sgen_marksweep_init (&major_collector);
4872         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed")) {
4873                 sgen_marksweep_fixed_init (&major_collector);
4874         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-par")) {
4875                 sgen_marksweep_par_init (&major_collector);
4876         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) {
4877                 sgen_marksweep_fixed_par_init (&major_collector);
4878         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4879                 sgen_marksweep_conc_init (&major_collector);
4880         } else {
4881                 fprintf (stderr, "Unknown major collector `%s'.\n", major_collector_opt);
4882                 exit (1);
4883         }
4884
4885 #ifdef SGEN_HAVE_CARDTABLE
4886         use_cardtable = major_collector.supports_cardtable;
4887 #else
4888         use_cardtable = FALSE;
4889 #endif
4890
4891         num_workers = mono_cpu_count ();
4892         g_assert (num_workers > 0);
4893         if (num_workers > 16)
4894                 num_workers = 16;
4895
4896         ///* Keep this the default for now */
4897         /* Precise marking is broken on all supported targets. Disable until fixed. */
4898         conservative_stack_mark = TRUE;
4899
4900         sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4901
4902         if (opts) {
4903                 for (ptr = opts; *ptr; ++ptr) {
4904                         char *opt = *ptr;
4905                         if (g_str_has_prefix (opt, "major="))
4906                                 continue;
4907                         if (g_str_has_prefix (opt, "minor="))
4908                                 continue;
4909                         if (g_str_has_prefix (opt, "wbarrier=")) {
4910                                 opt = strchr (opt, '=') + 1;
4911                                 if (strcmp (opt, "remset") == 0) {
4912                                         if (major_collector.is_concurrent) {
4913                                                 fprintf (stderr, "The concurrent collector does not support the SSB write barrier.\n");
4914                                                 exit (1);
4915                                         }
4916                                         use_cardtable = FALSE;
4917                                 } else if (strcmp (opt, "cardtable") == 0) {
4918                                         if (!use_cardtable) {
4919                                                 if (major_collector.supports_cardtable)
4920                                                         fprintf (stderr, "The cardtable write barrier is not supported on this platform.\n");
4921                                                 else
4922                                                         fprintf (stderr, "The major collector does not support the cardtable write barrier.\n");
4923                                                 exit (1);
4924                                         }
4925                                 } else {
4926                                         fprintf (stderr, "wbarrier must either be `remset' or `cardtable'.");
4927                                         exit (1);
4928                                 }
4929                                 continue;
4930                         }
4931                         if (g_str_has_prefix (opt, "max-heap-size=")) {
4932                                 opt = strchr (opt, '=') + 1;
4933                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap)) {
4934                                         if ((max_heap & (mono_pagesize () - 1))) {
4935                                                 fprintf (stderr, "max-heap-size size must be a multiple of %d.\n", mono_pagesize ());
4936                                                 exit (1);
4937                                         }
4938                                 } else {
4939                                         fprintf (stderr, "max-heap-size must be an integer.\n");
4940                                         exit (1);
4941                                 }
4942                                 continue;
4943                         }
4944                         if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4945                                 opt = strchr (opt, '=') + 1;
4946                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4947                                         if (soft_limit <= 0) {
4948                                                 fprintf (stderr, "soft-heap-limit must be positive.\n");
4949                                                 exit (1);
4950                                         }
4951                                 } else {
4952                                         fprintf (stderr, "soft-heap-limit must be an integer.\n");
4953                                         exit (1);
4954                                 }
4955                                 continue;
4956                         }
4957                         if (g_str_has_prefix (opt, "workers=")) {
4958                                 long val;
4959                                 char *endptr;
4960                                 if (!major_collector.is_parallel) {
4961                                         fprintf (stderr, "The workers= option can only be used for parallel collectors.");
4962                                         exit (1);
4963                                 }
4964                                 opt = strchr (opt, '=') + 1;
4965                                 val = strtol (opt, &endptr, 10);
4966                                 if (!*opt || *endptr) {
4967                                         fprintf (stderr, "Cannot parse the workers= option value.");
4968                                         exit (1);
4969                                 }
4970                                 if (val <= 0 || val > 16) {
4971                                         fprintf (stderr, "The number of workers must be in the range 1 to 16.");
4972                                         exit (1);
4973                                 }
4974                                 num_workers = (int)val;
4975                                 continue;
4976                         }
4977                         if (g_str_has_prefix (opt, "stack-mark=")) {
4978                                 opt = strchr (opt, '=') + 1;
4979                                 if (!strcmp (opt, "precise")) {
4980                                         conservative_stack_mark = FALSE;
4981                                 } else if (!strcmp (opt, "conservative")) {
4982                                         conservative_stack_mark = TRUE;
4983                                 } else {
4984                                         fprintf (stderr, "Invalid value '%s' for stack-mark= option, possible values are: 'precise', 'conservative'.\n", opt);
4985                                         exit (1);
4986                                 }
4987                                 continue;
4988                         }
4989                         if (g_str_has_prefix (opt, "bridge=")) {
4990                                 opt = strchr (opt, '=') + 1;
4991                                 sgen_register_test_bridge_callbacks (g_strdup (opt));
4992                                 continue;
4993                         }
4994 #ifdef USER_CONFIG
4995                         if (g_str_has_prefix (opt, "nursery-size=")) {
4996                                 long val;
4997                                 opt = strchr (opt, '=') + 1;
4998                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
4999                                         sgen_nursery_size = val;
5000 #ifdef SGEN_ALIGN_NURSERY
5001                                         if ((val & (val - 1))) {
5002                                                 fprintf (stderr, "The nursery size must be a power of two.\n");
5003                                                 exit (1);
5004                                         }
5005
5006                                         if (val < SGEN_MAX_NURSERY_WASTE) {
5007                                                 fprintf (stderr, "The nursery size must be at least %d bytes.\n", SGEN_MAX_NURSERY_WASTE);
5008                                                 exit (1);
5009                                         }
5010
5011                                         sgen_nursery_bits = 0;
5012                                         while (1 << (++ sgen_nursery_bits) != sgen_nursery_size)
5013                                                 ;
5014 #endif
5015                                 } else {
5016                                         fprintf (stderr, "nursery-size must be an integer.\n");
5017                                         exit (1);
5018                                 }
5019                                 continue;
5020                         }
5021 #endif
5022                         if (g_str_has_prefix (opt, "save-target-ratio=")) {
5023                                 char *endptr;
5024                                 opt = strchr (opt, '=') + 1;
5025                                 save_target = strtod (opt, &endptr);
5026                                 if (endptr == opt) {
5027                                         fprintf (stderr, "save-target-ratio must be a number.");
5028                                         exit (1);
5029                                 }
5030                                 if (save_target < SGEN_MIN_SAVE_TARGET_RATIO || save_target > SGEN_MAX_SAVE_TARGET_RATIO) {
5031                                         fprintf (stderr, "save-target-ratio must be between %.2f - %.2f.", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
5032                                         exit (1);
5033                                 }
5034                                 continue;
5035                         }
5036                         if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
5037                                 char *endptr;
5038                                 opt = strchr (opt, '=') + 1;
5039
5040                                 allowance_ratio = strtod (opt, &endptr);
5041                                 if (endptr == opt) {
5042                                         fprintf (stderr, "save-target-ratio must be a number.");
5043                                         exit (1);
5044                                 }
5045                                 if (allowance_ratio < SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO || allowance_ratio > SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO) {
5046                                         fprintf (stderr, "default-allowance-ratio must be between %.2f - %.2f.", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO);
5047                                         exit (1);
5048                                 }
5049                                 continue;
5050                         }
5051
5052                         if (!strcmp (opt, "cementing")) {
5053                                 cement_enabled = TRUE;
5054                                 continue;
5055                         }
5056                         if (!strcmp (opt, "no-cementing")) {
5057                                 cement_enabled = FALSE;
5058                                 continue;
5059                         }
5060
5061                         if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
5062                                 continue;
5063
5064                         if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
5065                                 continue;
5066
5067                         fprintf (stderr, "MONO_GC_PARAMS must be a comma-delimited list of one or more of the following:\n");
5068                         fprintf (stderr, "  max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5069                         fprintf (stderr, "  soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
5070                         fprintf (stderr, "  nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5071                         fprintf (stderr, "  major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-conc', `marksweep-par', 'marksweep-fixed' or 'marksweep-fixed-par')\n");
5072                         fprintf (stderr, "  minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
5073                         fprintf (stderr, "  wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
5074                         fprintf (stderr, "  stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
5075                         fprintf (stderr, "  [no-]cementing\n");
5076                         if (major_collector.print_gc_param_usage)
5077                                 major_collector.print_gc_param_usage ();
5078                         if (sgen_minor_collector.print_gc_param_usage)
5079                                 sgen_minor_collector.print_gc_param_usage ();
5080                         fprintf (stderr, " Experimental options:\n");
5081                         fprintf (stderr, "  save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
5082                         fprintf (stderr, "  default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
5083                         exit (1);
5084                 }
5085                 g_strfreev (opts);
5086         }
5087
5088         if (major_collector.is_parallel)
5089                 sgen_workers_init (num_workers);
5090         else if (major_collector.is_concurrent)
5091                 sgen_workers_init (1);
5092
5093         if (major_collector_opt)
5094                 g_free (major_collector_opt);
5095
5096         if (minor_collector_opt)
5097                 g_free (minor_collector_opt);
5098
5099         alloc_nursery ();
5100
5101         sgen_cement_init (cement_enabled);
5102
5103         if ((env = getenv ("MONO_GC_DEBUG"))) {
5104                 opts = g_strsplit (env, ",", -1);
5105                 for (ptr = opts; ptr && *ptr; ptr ++) {
5106                         char *opt = *ptr;
5107                         if (opt [0] >= '0' && opt [0] <= '9') {
5108                                 gc_debug_level = atoi (opt);
5109                                 opt++;
5110                                 if (opt [0] == ':')
5111                                         opt++;
5112                                 if (opt [0]) {
5113 #ifdef HOST_WIN32
5114                                         char *rf = g_strdup_printf ("%s.%d", opt, GetCurrentProcessId ());
5115 #else
5116                                         char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
5117 #endif
5118                                         gc_debug_file = fopen (rf, "wb");
5119                                         if (!gc_debug_file)
5120                                                 gc_debug_file = stderr;
5121                                         g_free (rf);
5122                                 }
5123                         } else if (!strcmp (opt, "print-allowance")) {
5124                                 debug_print_allowance = TRUE;
5125                         } else if (!strcmp (opt, "print-pinning")) {
5126                                 do_pin_stats = TRUE;
5127                         } else if (!strcmp (opt, "verify-before-allocs")) {
5128                                 verify_before_allocs = 1;
5129                                 has_per_allocation_action = TRUE;
5130                         } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
5131                                 char *arg = strchr (opt, '=') + 1;
5132                                 verify_before_allocs = atoi (arg);
5133                                 has_per_allocation_action = TRUE;
5134                         } else if (!strcmp (opt, "collect-before-allocs")) {
5135                                 collect_before_allocs = 1;
5136                                 has_per_allocation_action = TRUE;
5137                         } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
5138                                 char *arg = strchr (opt, '=') + 1;
5139                                 has_per_allocation_action = TRUE;
5140                                 collect_before_allocs = atoi (arg);
5141                         } else if (!strcmp (opt, "verify-before-collections")) {
5142                                 whole_heap_check_before_collection = TRUE;
5143                         } else if (!strcmp (opt, "check-at-minor-collections")) {
5144                                 consistency_check_at_minor_collection = TRUE;
5145                                 nursery_clear_policy = CLEAR_AT_GC;
5146                         } else if (!strcmp (opt, "check-mark-bits")) {
5147                                 check_mark_bits_after_major_collection = TRUE;
5148                         } else if (!strcmp (opt, "check-nursery-pinned")) {
5149                                 check_nursery_objects_pinned = TRUE;
5150                         } else if (!strcmp (opt, "xdomain-checks")) {
5151                                 xdomain_checks = TRUE;
5152                         } else if (!strcmp (opt, "clear-at-gc")) {
5153                                 nursery_clear_policy = CLEAR_AT_GC;
5154                         } else if (!strcmp (opt, "clear-nursery-at-gc")) {
5155                                 nursery_clear_policy = CLEAR_AT_GC;
5156                         } else if (!strcmp (opt, "check-scan-starts")) {
5157                                 do_scan_starts_check = TRUE;
5158                         } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
5159                                 do_verify_nursery = TRUE;
5160                         } else if (!strcmp (opt, "check-concurrent")) {
5161                                 if (!major_collector.is_concurrent) {
5162                                         fprintf (stderr, "Error: check-concurrent only world with concurrent major collectors.\n");
5163                                         exit (1);
5164                                 }
5165                                 do_concurrent_checks = TRUE;
5166                         } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
5167                                 do_dump_nursery_content = TRUE;
5168                         } else if (!strcmp (opt, "no-managed-allocator")) {
5169                                 sgen_set_use_managed_allocator (FALSE);
5170                         } else if (!strcmp (opt, "disable-minor")) {
5171                                 disable_minor_collections = TRUE;
5172                         } else if (!strcmp (opt, "disable-major")) {
5173                                 disable_major_collections = TRUE;
5174                         } else if (g_str_has_prefix (opt, "heap-dump=")) {
5175                                 char *filename = strchr (opt, '=') + 1;
5176                                 nursery_clear_policy = CLEAR_AT_GC;
5177                                 heap_dump_file = fopen (filename, "w");
5178                                 if (heap_dump_file) {
5179                                         fprintf (heap_dump_file, "<sgen-dump>\n");
5180                                         do_pin_stats = TRUE;
5181                                 }
5182 #ifdef SGEN_BINARY_PROTOCOL
5183                         } else if (g_str_has_prefix (opt, "binary-protocol=")) {
5184                                 char *filename = strchr (opt, '=') + 1;
5185                                 binary_protocol_init (filename);
5186                                 if (use_cardtable)
5187                                         fprintf (stderr, "Warning: Cardtable write barriers will not be binary-protocolled.\n");
5188 #endif
5189                         } else {
5190                                 fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
5191                                 fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
5192                                 fprintf (stderr, "Valid options are:\n");
5193                                 fprintf (stderr, "  collect-before-allocs[=<n>]\n");
5194                                 fprintf (stderr, "  verify-before-allocs[=<n>]\n");
5195                                 fprintf (stderr, "  check-at-minor-collections\n");
5196                                 fprintf (stderr, "  check-mark-bits\n");
5197                                 fprintf (stderr, "  check-nursery-pinned\n");
5198                                 fprintf (stderr, "  verify-before-collections\n");
5199                                 fprintf (stderr, "  verify-nursery-at-minor-gc\n");
5200                                 fprintf (stderr, "  dump-nursery-at-minor-gc\n");
5201                                 fprintf (stderr, "  disable-minor\n");
5202                                 fprintf (stderr, "  disable-major\n");
5203                                 fprintf (stderr, "  xdomain-checks\n");
5204                                 fprintf (stderr, "  check-concurrent\n");
5205                                 fprintf (stderr, "  clear-at-gc\n");
5206                                 fprintf (stderr, "  clear-nursery-at-gc\n");
5207                                 fprintf (stderr, "  check-scan-starts\n");
5208                                 fprintf (stderr, "  no-managed-allocator\n");
5209                                 fprintf (stderr, "  print-allowance\n");
5210                                 fprintf (stderr, "  print-pinning\n");
5211                                 fprintf (stderr, "  heap-dump=<filename>\n");
5212 #ifdef SGEN_BINARY_PROTOCOL
5213                                 fprintf (stderr, "  binary-protocol=<filename>\n");
5214 #endif
5215                                 exit (1);
5216                         }
5217                 }
5218                 g_strfreev (opts);
5219         }
5220
5221         if (major_collector.is_parallel) {
5222                 if (heap_dump_file) {
5223                         fprintf (stderr, "Error: Cannot do heap dump with the parallel collector.\n");
5224                         exit (1);
5225                 }
5226                 if (do_pin_stats) {
5227                         fprintf (stderr, "Error: Cannot gather pinning statistics with the parallel collector.\n");
5228                         exit (1);
5229                 }
5230         }
5231
5232         if (major_collector.post_param_init)
5233                 major_collector.post_param_init (&major_collector);
5234
5235         sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
5236
5237         memset (&remset, 0, sizeof (remset));
5238
5239 #ifdef SGEN_HAVE_CARDTABLE
5240         if (use_cardtable)
5241                 sgen_card_table_init (&remset);
5242         else
5243 #endif
5244                 sgen_ssb_init (&remset);
5245
5246         if (remset.register_thread)
5247                 remset.register_thread (mono_thread_info_current ());
5248
5249         gc_initialized = 1;
5250 }
5251
5252 const char *
5253 mono_gc_get_gc_name (void)
5254 {
5255         return "sgen";
5256 }
5257
5258 static MonoMethod *write_barrier_method;
5259
5260 gboolean
5261 sgen_is_critical_method (MonoMethod *method)
5262 {
5263         return (method == write_barrier_method || sgen_is_managed_allocator (method));
5264 }
5265
5266 gboolean
5267 sgen_has_critical_method (void)
5268 {
5269         return write_barrier_method || sgen_has_managed_allocator ();
5270 }
5271
5272 #ifndef DISABLE_JIT
5273
5274 static void
5275 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5276 {
5277         memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5278 #ifdef SGEN_ALIGN_NURSERY
5279         // if (ptr_in_nursery (ptr)) return;
5280         /*
5281          * Masking out the bits might be faster, but we would have to use 64 bit
5282          * immediates, which might be slower.
5283          */
5284         mono_mb_emit_ldarg (mb, 0);
5285         mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5286         mono_mb_emit_byte (mb, CEE_SHR_UN);
5287         mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5288         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5289
5290         if (!major_collector.is_concurrent) {
5291                 // if (!ptr_in_nursery (*ptr)) return;
5292                 mono_mb_emit_ldarg (mb, 0);
5293                 mono_mb_emit_byte (mb, CEE_LDIND_I);
5294                 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5295                 mono_mb_emit_byte (mb, CEE_SHR_UN);
5296                 mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5297                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5298         }
5299 #else
5300         int label_continue1, label_continue2;
5301         int dereferenced_var;
5302
5303         // if (ptr < (sgen_get_nursery_start ())) goto continue;
5304         mono_mb_emit_ldarg (mb, 0);
5305         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5306         label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5307
5308         // if (ptr >= sgen_get_nursery_end ())) goto continue;
5309         mono_mb_emit_ldarg (mb, 0);
5310         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5311         label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5312
5313         // Otherwise return
5314         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5315
5316         // continue:
5317         mono_mb_patch_branch (mb, label_continue_1);
5318         mono_mb_patch_branch (mb, label_continue_2);
5319
5320         // Dereference and store in local var
5321         dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5322         mono_mb_emit_ldarg (mb, 0);
5323         mono_mb_emit_byte (mb, CEE_LDIND_I);
5324         mono_mb_emit_stloc (mb, dereferenced_var);
5325
5326         if (!major_collector.is_concurrent) {
5327                 // if (*ptr < sgen_get_nursery_start ()) return;
5328                 mono_mb_emit_ldloc (mb, dereferenced_var);
5329                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5330                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5331
5332                 // if (*ptr >= sgen_get_nursery_end ()) return;
5333                 mono_mb_emit_ldloc (mb, dereferenced_var);
5334                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5335                 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5336         }
5337 #endif
5338 }
5339 #endif
5340
5341 MonoMethod*
5342 mono_gc_get_write_barrier (void)
5343 {
5344         MonoMethod *res;
5345         MonoMethodBuilder *mb;
5346         MonoMethodSignature *sig;
5347 #ifdef MANAGED_WBARRIER
5348         int i, nursery_check_labels [3];
5349         int label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
5350         int buffer_var, buffer_index_var, dummy_var;
5351
5352 #ifdef HAVE_KW_THREAD
5353         int stack_end_offset = -1, store_remset_buffer_offset = -1;
5354         int store_remset_buffer_index_offset = -1, store_remset_buffer_index_addr_offset = -1;
5355
5356         MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5357         g_assert (stack_end_offset != -1);
5358         MONO_THREAD_VAR_OFFSET (store_remset_buffer, store_remset_buffer_offset);
5359         g_assert (store_remset_buffer_offset != -1);
5360         MONO_THREAD_VAR_OFFSET (store_remset_buffer_index, store_remset_buffer_index_offset);
5361         g_assert (store_remset_buffer_index_offset != -1);
5362         MONO_THREAD_VAR_OFFSET (store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
5363         g_assert (store_remset_buffer_index_addr_offset != -1);
5364 #endif
5365 #endif
5366
5367         // FIXME: Maybe create a separate version for ctors (the branch would be
5368         // correctly predicted more times)
5369         if (write_barrier_method)
5370                 return write_barrier_method;
5371
5372         /* Create the IL version of mono_gc_barrier_generic_store () */
5373         sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5374         sig->ret = &mono_defaults.void_class->byval_arg;
5375         sig->params [0] = &mono_defaults.int_class->byval_arg;
5376
5377         mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5378
5379 #ifndef DISABLE_JIT
5380 #ifdef MANAGED_WBARRIER
5381         if (use_cardtable) {
5382                 emit_nursery_check (mb, nursery_check_labels);
5383                 /*
5384                 addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5385                 *addr = 1;
5386
5387                 sgen_cardtable:
5388                         LDC_PTR sgen_cardtable
5389
5390                 address >> CARD_BITS
5391                         LDARG_0
5392                         LDC_I4 CARD_BITS
5393                         SHR_UN
5394                 if (SGEN_HAVE_OVERLAPPING_CARDS) {
5395                         LDC_PTR card_table_mask
5396                         AND
5397                 }
5398                 AND
5399                 ldc_i4_1
5400                 stind_i1
5401                 */
5402                 mono_mb_emit_ptr (mb, sgen_cardtable);
5403                 mono_mb_emit_ldarg (mb, 0);
5404                 mono_mb_emit_icon (mb, CARD_BITS);
5405                 mono_mb_emit_byte (mb, CEE_SHR_UN);
5406 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5407                 mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5408                 mono_mb_emit_byte (mb, CEE_AND);
5409 #endif
5410                 mono_mb_emit_byte (mb, CEE_ADD);
5411                 mono_mb_emit_icon (mb, 1);
5412                 mono_mb_emit_byte (mb, CEE_STIND_I1);
5413
5414                 // return;
5415                 for (i = 0; i < 3; ++i) {
5416                         if (nursery_check_labels [i])
5417                                 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5418                 }
5419                 mono_mb_emit_byte (mb, CEE_RET);
5420         } else if (mono_runtime_has_tls_get ()) {
5421                 emit_nursery_check (mb, nursery_check_labels);
5422
5423                 // if (ptr >= stack_end) goto need_wb;
5424                 mono_mb_emit_ldarg (mb, 0);
5425                 EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
5426                 label_need_wb = mono_mb_emit_branch (mb, CEE_BGE_UN);
5427
5428                 // if (ptr >= stack_start) return;
5429                 dummy_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5430                 mono_mb_emit_ldarg (mb, 0);
5431                 mono_mb_emit_ldloc_addr (mb, dummy_var);
5432                 label_no_wb_3 = mono_mb_emit_branch (mb, CEE_BGE_UN);
5433
5434                 // need_wb:
5435                 mono_mb_patch_branch (mb, label_need_wb);
5436
5437                 // buffer = STORE_REMSET_BUFFER;
5438                 buffer_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5439                 EMIT_TLS_ACCESS (mb, store_remset_buffer, store_remset_buffer_offset);
5440                 mono_mb_emit_stloc (mb, buffer_var);
5441
5442                 // buffer_index = STORE_REMSET_BUFFER_INDEX;
5443                 buffer_index_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5444                 EMIT_TLS_ACCESS (mb, store_remset_buffer_index, store_remset_buffer_index_offset);
5445                 mono_mb_emit_stloc (mb, buffer_index_var);
5446
5447                 // if (buffer [buffer_index] == ptr) return;
5448                 mono_mb_emit_ldloc (mb, buffer_var);
5449                 mono_mb_emit_ldloc (mb, buffer_index_var);
5450                 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
5451                 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
5452                 mono_mb_emit_byte (mb, CEE_SHL);
5453                 mono_mb_emit_byte (mb, CEE_ADD);
5454                 mono_mb_emit_byte (mb, CEE_LDIND_I);
5455                 mono_mb_emit_ldarg (mb, 0);
5456                 label_no_wb_4 = mono_mb_emit_branch (mb, CEE_BEQ);
5457
5458                 // ++buffer_index;
5459                 mono_mb_emit_ldloc (mb, buffer_index_var);
5460                 mono_mb_emit_icon (mb, 1);
5461                 mono_mb_emit_byte (mb, CEE_ADD);
5462                 mono_mb_emit_stloc (mb, buffer_index_var);
5463
5464                 // if (buffer_index >= STORE_REMSET_BUFFER_SIZE) goto slow_path;
5465                 mono_mb_emit_ldloc (mb, buffer_index_var);
5466                 mono_mb_emit_icon (mb, STORE_REMSET_BUFFER_SIZE);
5467                 label_slow_path = mono_mb_emit_branch (mb, CEE_BGE);
5468
5469                 // buffer [buffer_index] = ptr;
5470                 mono_mb_emit_ldloc (mb, buffer_var);
5471                 mono_mb_emit_ldloc (mb, buffer_index_var);
5472                 g_assert (sizeof (gpointer) == 4 || sizeof (gpointer) == 8);
5473                 mono_mb_emit_icon (mb, sizeof (gpointer) == 4 ? 2 : 3);
5474                 mono_mb_emit_byte (mb, CEE_SHL);
5475                 mono_mb_emit_byte (mb, CEE_ADD);
5476                 mono_mb_emit_ldarg (mb, 0);
5477                 mono_mb_emit_byte (mb, CEE_STIND_I);
5478
5479                 // STORE_REMSET_BUFFER_INDEX = buffer_index;
5480                 EMIT_TLS_ACCESS (mb, store_remset_buffer_index_addr, store_remset_buffer_index_addr_offset);
5481                 mono_mb_emit_ldloc (mb, buffer_index_var);
5482                 mono_mb_emit_byte (mb, CEE_STIND_I);
5483
5484                 // return;
5485                 for (i = 0; i < 3; ++i) {
5486                         if (nursery_check_labels [i])
5487                                 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5488                 }
5489                 mono_mb_patch_branch (mb, label_no_wb_3);
5490                 mono_mb_patch_branch (mb, label_no_wb_4);
5491                 mono_mb_emit_byte (mb, CEE_RET);
5492
5493                 // slow path
5494                 mono_mb_patch_branch (mb, label_slow_path);
5495
5496                 mono_mb_emit_ldarg (mb, 0);
5497                 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5498                 mono_mb_emit_byte (mb, CEE_RET);
5499         } else
5500 #endif
5501         {
5502                 mono_mb_emit_ldarg (mb, 0);
5503                 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5504                 mono_mb_emit_byte (mb, CEE_RET);
5505         }
5506
5507 #endif
5508         res = mono_mb_create_method (mb, sig, 16);
5509         mono_mb_free (mb);
5510
5511         mono_loader_lock ();
5512         if (write_barrier_method) {
5513                 /* Already created */
5514                 mono_free_method (res);
5515         } else {
5516                 /* double-checked locking */
5517                 mono_memory_barrier ();
5518                 write_barrier_method = res;
5519         }
5520         mono_loader_unlock ();
5521
5522         return write_barrier_method;
5523 }
5524
5525 char*
5526 mono_gc_get_description (void)
5527 {
5528         return g_strdup ("sgen");
5529 }
5530
5531 void
5532 mono_gc_set_desktop_mode (void)
5533 {
5534 }
5535
5536 gboolean
5537 mono_gc_is_moving (void)
5538 {
5539         return TRUE;
5540 }
5541
5542 gboolean
5543 mono_gc_is_disabled (void)
5544 {
5545         return FALSE;
5546 }
5547
5548 #ifdef HOST_WIN32
5549 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5550 {
5551         return TRUE;
5552 }
5553 #endif
5554
5555 NurseryClearPolicy
5556 sgen_get_nursery_clear_policy (void)
5557 {
5558         return nursery_clear_policy;
5559 }
5560
5561 MonoVTable*
5562 sgen_get_array_fill_vtable (void)
5563 {
5564         if (!array_fill_vtable) {
5565                 static MonoClass klass;
5566                 static MonoVTable vtable;
5567                 gsize bmap;
5568
5569                 MonoDomain *domain = mono_get_root_domain ();
5570                 g_assert (domain);
5571
5572                 klass.element_class = mono_defaults.byte_class;
5573                 klass.rank = 1;
5574                 klass.instance_size = sizeof (MonoArray);
5575                 klass.sizes.element_size = 1;
5576                 klass.name = "array_filler_type";
5577
5578                 vtable.klass = &klass;
5579                 bmap = 0;
5580                 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5581                 vtable.rank = 1;
5582
5583                 array_fill_vtable = &vtable;
5584         }
5585         return array_fill_vtable;
5586 }
5587
5588 void
5589 sgen_gc_lock (void)
5590 {
5591         LOCK_GC;
5592 }
5593
5594 void
5595 sgen_gc_unlock (void)
5596 {
5597         UNLOCK_GC;
5598 }
5599
5600 void
5601 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5602 {
5603         major_collector.iterate_live_block_ranges (callback);
5604 }
5605
5606 void
5607 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5608 {
5609         major_collector.scan_card_table (FALSE, queue);
5610 }
5611
5612 SgenMajorCollector*
5613 sgen_get_major_collector (void)
5614 {
5615         return &major_collector;
5616 }
5617
5618 void mono_gc_set_skip_thread (gboolean skip)
5619 {
5620         SgenThreadInfo *info = mono_thread_info_current ();
5621
5622         LOCK_GC;
5623         info->gc_disabled = skip;
5624         UNLOCK_GC;
5625 }
5626
5627 SgenRemeberedSet*
5628 sgen_get_remset (void)
5629 {
5630         return &remset;
5631 }
5632
5633 guint
5634 mono_gc_get_vtable_bits (MonoClass *class)
5635 {
5636         if (sgen_need_bridge_processing () && sgen_is_bridge_class (class))
5637                 return SGEN_GC_BIT_BRIDGE_OBJECT;
5638         return 0;
5639 }
5640
5641 void
5642 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5643 {
5644         // FIXME:
5645 }
5646
5647
5648 void
5649 sgen_check_whole_heap_stw (void)
5650 {
5651         sgen_stop_world (0);
5652         sgen_clear_nursery_fragments ();
5653         sgen_check_whole_heap (FALSE);
5654         sgen_restart_world (0, NULL);
5655 }
5656
5657 void
5658 sgen_gc_event_moves (void)
5659 {
5660         if (moved_objects_idx) {
5661                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5662                 moved_objects_idx = 0;
5663         }
5664 }
5665
5666 #endif /* HAVE_SGEN_GC */