mono/metadata/sgen-gc.c

   1 /*
   2  * sgen-gc.c: Simple generational GC.
   3  *
   4  * Author:
   5  *      Paolo Molaro (lupus@ximian.com)
   6  *  Rodrigo Kumpera (kumpera@gmail.com)
   7  *
   8  * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
   9  * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
  10  *
  11  * Thread start/stop adapted from Boehm's GC:
  12  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  13  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  14  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  15  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  16  * Copyright 2001-2003 Ximian, Inc
  17  * Copyright 2003-2010 Novell, Inc.
  18  * Copyright 2011 Xamarin, Inc.
  19  * Copyright (C) 2012 Xamarin Inc
  20  *
  21  * This library is free software; you can redistribute it and/or
  22  * modify it under the terms of the GNU Library General Public
  23  * License 2.0 as published by the Free Software Foundation;
  24  *
  25  * This library is distributed in the hope that it will be useful,
  26  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  28  * Library General Public License for more details.
  29  *
  30  * You should have received a copy of the GNU Library General Public
  31  * License 2.0 along with this library; if not, write to the Free
  32  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  33  *
  34  * Important: allocation provides always zeroed memory, having to do
  35  * a memset after allocation is deadly for performance.
  36  * Memory usage at startup is currently as follows:
  37  * 64 KB pinned space
  38  * 64 KB internal space
  39  * size of nursery
  40  * We should provide a small memory config with half the sizes
  41  *
  42  * We currently try to make as few mono assumptions as possible:
  43  * 1) 2-word header with no GC pointers in it (first vtable, second to store the
  44  *    forwarding ptr)
  45  * 2) gc descriptor is the second word in the vtable (first word in the class)
  46  * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
  47  * 4) there is a function to get an object's size and the number of
  48  *    elements in an array.
  49  * 5) we know the special way bounds are allocated for complex arrays
  50  * 6) we know about proxies and how to treat them when domains are unloaded
  51  *
  52  * Always try to keep stack usage to a minimum: no recursive behaviour
  53  * and no large stack allocs.
  54  *
  55  * General description.
  56  * Objects are initially allocated in a nursery using a fast bump-pointer technique.
  57  * When the nursery is full we start a nursery collection: this is performed with a
  58  * copying GC.
  59  * When the old generation is full we start a copying GC of the old generation as well:
  60  * this will be changed to mark&sweep with copying when fragmentation becomes to severe
  61  * in the future.  Maybe we'll even do both during the same collection like IMMIX.
  62  *
  63  * The things that complicate this description are:
  64  * *) pinned objects: we can't move them so we need to keep track of them
  65  * *) no precise info of the thread stacks and registers: we need to be able to
  66  *    quickly find the objects that may be referenced conservatively and pin them
  67  *    (this makes the first issues more important)
  68  * *) large objects are too expensive to be dealt with using copying GC: we handle them
  69  *    with mark/sweep during major collections
  70  * *) some objects need to not move even if they are small (interned strings, Type handles):
  71  *    we use mark/sweep for them, too: they are not allocated in the nursery, but inside
  72  *    PinnedChunks regions
  73  */
  74
  75 /*
  76  * TODO:
  77
  78  *) we could have a function pointer in MonoClass to implement
  79   customized write barriers for value types
  80
  81  *) investigate the stuff needed to advance a thread to a GC-safe
  82   point (single-stepping, read from unmapped memory etc) and implement it.
  83   This would enable us to inline allocations and write barriers, for example,
  84   or at least parts of them, like the write barrier checks.
  85   We may need this also for handling precise info on stacks, even simple things
  86   as having uninitialized data on the stack and having to wait for the prolog
  87   to zero it. Not an issue for the last frame that we scan conservatively.
  88   We could always not trust the value in the slots anyway.
  89
  90  *) modify the jit to save info about references in stack locations:
  91   this can be done just for locals as a start, so that at least
  92   part of the stack is handled precisely.
  93
  94  *) test/fix endianess issues
  95
  96  *) Implement a card table as the write barrier instead of remembered
  97     sets?  Card tables are not easy to implement with our current
  98     memory layout.  We have several different kinds of major heap
  99     objects: Small objects in regular blocks, small objects in pinned
 100     chunks and LOS objects.  If we just have a pointer we have no way
 101     to tell which kind of object it points into, therefore we cannot
 102     know where its card table is.  The least we have to do to make
 103     this happen is to get rid of write barriers for indirect stores.
 104     (See next item)
 105
 106  *) Get rid of write barriers for indirect stores.  We can do this by
 107     telling the GC to wbarrier-register an object once we do an ldloca
 108     or ldelema on it, and to unregister it once it's not used anymore
 109     (it can only travel downwards on the stack).  The problem with
 110     unregistering is that it needs to happen eventually no matter
 111     what, even if exceptions are thrown, the thread aborts, etc.
 112     Rodrigo suggested that we could do only the registering part and
 113     let the collector find out (pessimistically) when it's safe to
 114     unregister, namely when the stack pointer of the thread that
 115     registered the object is higher than it was when the registering
 116     happened.  This might make for a good first implementation to get
 117     some data on performance.
 118
 119  *) Some sort of blacklist support?  Blacklists is a concept from the
 120     Boehm GC: if during a conservative scan we find pointers to an
 121     area which we might use as heap, we mark that area as unusable, so
 122     pointer retention by random pinning pointers is reduced.
 123
 124  *) experiment with max small object size (very small right now - 2kb,
 125     because it's tied to the max freelist size)
 126
 127   *) add an option to mmap the whole heap in one chunk: it makes for many
 128      simplifications in the checks (put the nursery at the top and just use a single
 129      check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
 130      not flexible (too much of the address space may be used by default or we can't
 131      increase the heap as needed) and we'd need a race-free mechanism to return memory
 132      back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
 133      was written to, munmap is needed, but the following mmap may not find the same segment
 134      free...)
 135
 136  *) memzero the major fragments after restarting the world and optionally a smaller
 137     chunk at a time
 138
 139  *) investigate having fragment zeroing threads
 140
 141  *) separate locks for finalization and other minor stuff to reduce
 142     lock contention
 143
 144  *) try a different copying order to improve memory locality
 145
 146  *) a thread abort after a store but before the write barrier will
 147     prevent the write barrier from executing
 148
 149  *) specialized dynamically generated markers/copiers
 150
 151  *) Dynamically adjust TLAB size to the number of threads.  If we have
 152     too many threads that do allocation, we might need smaller TLABs,
 153     and we might get better performance with larger TLABs if we only
 154     have a handful of threads.  We could sum up the space left in all
 155     assigned TLABs and if that's more than some percentage of the
 156     nursery size, reduce the TLAB size.
 157
 158  *) Explore placing unreachable objects on unused nursery memory.
 159         Instead of memset'ng a region to zero, place an int[] covering it.
 160         A good place to start is add_nursery_frag. The tricky thing here is
 161         placing those objects atomically outside of a collection.
 162
 163  *) Allocation should use asymmetric Dekker synchronization:
 164         http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
 165         This should help weak consistency archs.
 166  */
 167 #include "config.h"
 168 #ifdef HAVE_SGEN_GC
 169
 170 #ifdef __MACH__
 171 #undef _XOPEN_SOURCE
 172 #define _XOPEN_SOURCE
 173 #define _DARWIN_C_SOURCE
 174 #endif
 175
 176 #ifdef HAVE_UNISTD_H
 177 #include <unistd.h>
 178 #endif
 179 #ifdef HAVE_PTHREAD_H
 180 #include <pthread.h>
 181 #endif
 182 #ifdef HAVE_PTHREAD_NP_H
 183 #include <pthread_np.h>
 184 #endif
 185 #ifdef HAVE_SEMAPHORE_H
 186 #include <semaphore.h>
 187 #endif
 188 #include <stdio.h>
 189 #include <string.h>
 190 #include <signal.h>
 191 #include <errno.h>
 192 #include <assert.h>
 193
 194 #include "metadata/sgen-gc.h"
 195 #include "metadata/metadata-internals.h"
 196 #include "metadata/class-internals.h"
 197 #include "metadata/gc-internal.h"
 198 #include "metadata/object-internals.h"
 199 #include "metadata/threads.h"
 200 #include "metadata/sgen-cardtable.h"
 201 #include "metadata/sgen-protocol.h"
 202 #include "metadata/sgen-archdep.h"
 203 #include "metadata/sgen-bridge.h"
 204 #include "metadata/sgen-memory-governor.h"
 205 #include "metadata/sgen-hash-table.h"
 206 #include "metadata/mono-gc.h"
 207 #include "metadata/method-builder.h"
 208 #include "metadata/profiler-private.h"
 209 #include "metadata/monitor.h"
 210 #include "metadata/threadpool-internals.h"
 211 #include "metadata/mempool-internals.h"
 212 #include "metadata/marshal.h"
 213 #include "metadata/runtime.h"
 214 #include "metadata/sgen-cardtable.h"
 215 #include "metadata/sgen-pinning.h"
 216 #include "metadata/sgen-workers.h"
 217 #include "metadata/sgen-layout-stats.h"
 218 #include "utils/mono-mmap.h"
 219 #include "utils/mono-time.h"
 220 #include "utils/mono-semaphore.h"
 221 #include "utils/mono-counters.h"
 222 #include "utils/mono-proclib.h"
 223 #include "utils/mono-memory-model.h"
 224 #include "utils/mono-logger-internal.h"
 225 #include "utils/dtrace.h"
 226
 227 #include <mono/utils/mono-logger-internal.h>
 228 #include <mono/utils/memcheck.h>
 229
 230 #if defined(__MACH__)
 231 #include "utils/mach-support.h"
 232 #endif
 233
 234 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
 235         a = i,
 236
 237 enum {
 238 #include "mono/cil/opcode.def"
 239         CEE_LAST
 240 };
 241
 242 #undef OPDEF
 243
 244 #undef pthread_create
 245 #undef pthread_join
 246 #undef pthread_detach
 247
 248 /*
 249  * ######################################################################
 250  * ########  Types and constants used by the GC.
 251  * ######################################################################
 252  */
 253
 254 /* 0 means not initialized, 1 is initialized, -1 means in progress */
 255 static int gc_initialized = 0;
 256 /* If set, check if we need to do something every X allocations */
 257 gboolean has_per_allocation_action;
 258 /* If set, do a heap check every X allocation */
 259 guint32 verify_before_allocs = 0;
 260 /* If set, do a minor collection before every X allocation */
 261 guint32 collect_before_allocs = 0;
 262 /* If set, do a whole heap check before each collection */
 263 static gboolean whole_heap_check_before_collection = FALSE;
 264 /* If set, do a heap consistency check before each minor collection */
 265 static gboolean consistency_check_at_minor_collection = FALSE;
 266 /* If set, do a mod union consistency check before each finishing collection pause */
 267 static gboolean mod_union_consistency_check = FALSE;
 268 /* If set, check whether mark bits are consistent after major collections */
 269 static gboolean check_mark_bits_after_major_collection = FALSE;
 270 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
 271 static gboolean check_nursery_objects_pinned = FALSE;
 272 /* If set, do a few checks when the concurrent collector is used */
 273 static gboolean do_concurrent_checks = FALSE;
 274 /* If set, check that there are no references to the domain left at domain unload */
 275 static gboolean xdomain_checks = FALSE;
 276 /* If not null, dump the heap after each collection into this file */
 277 static FILE *heap_dump_file = NULL;
 278 /* If set, mark stacks conservatively, even if precise marking is possible */
 279 static gboolean conservative_stack_mark = FALSE;
 280 /* If set, do a plausibility check on the scan_starts before and after
 281    each collection */
 282 static gboolean do_scan_starts_check = FALSE;
 283 /*
 284  * If the major collector is concurrent and this is FALSE, we will
 285  * never initiate a synchronous major collection, unless requested via
 286  * GC.Collect().
 287  */
 288 static gboolean allow_synchronous_major = TRUE;
 289 static gboolean nursery_collection_is_parallel = FALSE;
 290 static gboolean disable_minor_collections = FALSE;
 291 static gboolean disable_major_collections = FALSE;
 292 gboolean do_pin_stats = FALSE;
 293 static gboolean do_verify_nursery = FALSE;
 294 static gboolean do_dump_nursery_content = FALSE;
 295
 296 #ifdef HEAVY_STATISTICS
 297 long long stat_objects_alloced_degraded = 0;
 298 long long stat_bytes_alloced_degraded = 0;
 299
 300 long long stat_copy_object_called_nursery = 0;
 301 long long stat_objects_copied_nursery = 0;
 302 long long stat_copy_object_called_major = 0;
 303 long long stat_objects_copied_major = 0;
 304
 305 long long stat_scan_object_called_nursery = 0;
 306 long long stat_scan_object_called_major = 0;
 307
 308 long long stat_slots_allocated_in_vain;
 309
 310 long long stat_nursery_copy_object_failed_from_space = 0;
 311 long long stat_nursery_copy_object_failed_forwarded = 0;
 312 long long stat_nursery_copy_object_failed_pinned = 0;
 313 long long stat_nursery_copy_object_failed_to_space = 0;
 314
 315 static int stat_wbarrier_add_to_global_remset = 0;
 316 static int stat_wbarrier_set_field = 0;
 317 static int stat_wbarrier_set_arrayref = 0;
 318 static int stat_wbarrier_arrayref_copy = 0;
 319 static int stat_wbarrier_generic_store = 0;
 320 static int stat_wbarrier_set_root = 0;
 321 static int stat_wbarrier_value_copy = 0;
 322 static int stat_wbarrier_object_copy = 0;
 323 #endif
 324
 325 int stat_minor_gcs = 0;
 326 int stat_major_gcs = 0;
 327
 328 static long long stat_pinned_objects = 0;
 329
 330 static long long time_minor_pre_collection_fragment_clear = 0;
 331 static long long time_minor_pinning = 0;
 332 static long long time_minor_scan_remsets = 0;
 333 static long long time_minor_scan_pinned = 0;
 334 static long long time_minor_scan_registered_roots = 0;
 335 static long long time_minor_scan_thread_data = 0;
 336 static long long time_minor_finish_gray_stack = 0;
 337 static long long time_minor_fragment_creation = 0;
 338
 339 static long long time_major_pre_collection_fragment_clear = 0;
 340 static long long time_major_pinning = 0;
 341 static long long time_major_scan_pinned = 0;
 342 static long long time_major_scan_registered_roots = 0;
 343 static long long time_major_scan_thread_data = 0;
 344 static long long time_major_scan_alloc_pinned = 0;
 345 static long long time_major_scan_finalized = 0;
 346 static long long time_major_scan_big_objects = 0;
 347 static long long time_major_finish_gray_stack = 0;
 348 static long long time_major_free_bigobjs = 0;
 349 static long long time_major_los_sweep = 0;
 350 static long long time_major_sweep = 0;
 351 static long long time_major_fragment_creation = 0;
 352
 353 int gc_debug_level = 0;
 354 FILE* gc_debug_file;
 355
 356 /*
 357 void
 358 mono_gc_flush_info (void)
 359 {
 360         fflush (gc_debug_file);
 361 }
 362 */
 363
 364 #define TV_DECLARE SGEN_TV_DECLARE
 365 #define TV_GETTIME SGEN_TV_GETTIME
 366 #define TV_ELAPSED SGEN_TV_ELAPSED
 367 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
 368
 369 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
 370
 371 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
 372
 373 #define object_is_forwarded     SGEN_OBJECT_IS_FORWARDED
 374 #define object_is_pinned        SGEN_OBJECT_IS_PINNED
 375 #define pin_object              SGEN_PIN_OBJECT
 376 #define unpin_object            SGEN_UNPIN_OBJECT
 377
 378 #define ptr_in_nursery sgen_ptr_in_nursery
 379
 380 #define LOAD_VTABLE     SGEN_LOAD_VTABLE
 381
 382 static const char*
 383 safe_name (void* obj)
 384 {
 385         MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
 386         return vt->klass->name;
 387 }
 388
 389 #define safe_object_get_size    sgen_safe_object_get_size
 390
 391 const char*
 392 sgen_safe_name (void* obj)
 393 {
 394         return safe_name (obj);
 395 }
 396
 397 /*
 398  * ######################################################################
 399  * ########  Global data.
 400  * ######################################################################
 401  */
 402 LOCK_DECLARE (gc_mutex);
 403
 404 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
 405
 406 static mword pagesize = 4096;
 407 int degraded_mode = 0;
 408
 409 static mword bytes_pinned_from_failed_allocation = 0;
 410
 411 GCMemSection *nursery_section = NULL;
 412 static mword lowest_heap_address = ~(mword)0;
 413 static mword highest_heap_address = 0;
 414
 415 LOCK_DECLARE (sgen_interruption_mutex);
 416 static LOCK_DECLARE (pin_queue_mutex);
 417
 418 #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex)
 419 #define UNLOCK_PIN_QUEUE mono_mutex_unlock (&pin_queue_mutex)
 420
 421 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
 422 struct _FinalizeReadyEntry {
 423         FinalizeReadyEntry *next;
 424         void *object;
 425 };
 426
 427 typedef struct _EphemeronLinkNode EphemeronLinkNode;
 428
 429 struct _EphemeronLinkNode {
 430         EphemeronLinkNode *next;
 431         char *array;
 432 };
 433
 434 typedef struct {
 435        void *key;
 436        void *value;
 437 } Ephemeron;
 438
 439 int current_collection_generation = -1;
 440 volatile gboolean concurrent_collection_in_progress = FALSE;
 441
 442 /* objects that are ready to be finalized */
 443 static FinalizeReadyEntry *fin_ready_list = NULL;
 444 static FinalizeReadyEntry *critical_fin_list = NULL;
 445
 446 static EphemeronLinkNode *ephemeron_list;
 447
 448 /* registered roots: the key to the hash is the root start address */
 449 /*
 450  * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
 451  */
 452 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
 453         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 454         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 455         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
 456 };
 457 static mword roots_size = 0; /* amount of memory in the root set */
 458
 459 #define GC_ROOT_NUM 32
 460 typedef struct {
 461         int count;              /* must be the first field */
 462         void *objects [GC_ROOT_NUM];
 463         int root_types [GC_ROOT_NUM];
 464         uintptr_t extra_info [GC_ROOT_NUM];
 465 } GCRootReport;
 466
 467 static void
 468 notify_gc_roots (GCRootReport *report)
 469 {
 470         if (!report->count)
 471                 return;
 472         mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
 473         report->count = 0;
 474 }
 475
 476 static void
 477 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
 478 {
 479         if (report->count == GC_ROOT_NUM)
 480                 notify_gc_roots (report);
 481         report->objects [report->count] = object;
 482         report->root_types [report->count] = rtype;
 483         report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
 484 }
 485
 486 MonoNativeTlsKey thread_info_key;
 487
 488 #ifdef HAVE_KW_THREAD
 489 __thread SgenThreadInfo *sgen_thread_info;
 490 __thread char *stack_end;
 491 #endif
 492
 493 /* The size of a TLAB */
 494 /* The bigger the value, the less often we have to go to the slow path to allocate a new
 495  * one, but the more space is wasted by threads not allocating much memory.
 496  * FIXME: Tune this.
 497  * FIXME: Make this self-tuning for each thread.
 498  */
 499 guint32 tlab_size = (1024 * 4);
 500
 501 #define MAX_SMALL_OBJ_SIZE      SGEN_MAX_SMALL_OBJ_SIZE
 502
 503 /* Functions supplied by the runtime to be called by the GC */
 504 static MonoGCCallbacks gc_callbacks;
 505
 506 #define ALLOC_ALIGN             SGEN_ALLOC_ALIGN
 507 #define ALLOC_ALIGN_BITS        SGEN_ALLOC_ALIGN_BITS
 508
 509 #define ALIGN_UP                SGEN_ALIGN_UP
 510
 511 #define MOVED_OBJECTS_NUM 64
 512 static void *moved_objects [MOVED_OBJECTS_NUM];
 513 static int moved_objects_idx = 0;
 514
 515 /* Vtable of the objects used to fill out nursery fragments before a collection */
 516 static MonoVTable *array_fill_vtable;
 517
 518 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
 519 MonoNativeThreadId main_gc_thread = NULL;
 520 #endif
 521
 522 /*Object was pinned during the current collection*/
 523 static mword objects_pinned;
 524
 525 /*
 526  * ######################################################################
 527  * ########  Macros and function declarations.
 528  * ######################################################################
 529  */
 530
 531 inline static void*
 532 align_pointer (void *ptr)
 533 {
 534         mword p = (mword)ptr;
 535         p += sizeof (gpointer) - 1;
 536         p &= ~ (sizeof (gpointer) - 1);
 537         return (void*)p;
 538 }
 539
 540 typedef SgenGrayQueue GrayQueue;
 541
 542 /* forward declarations */
 543 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
 544 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
 545 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
 546 static void report_finalizer_roots (void);
 547 static void report_registered_roots (void);
 548
 549 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
 550 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx);
 551 static void finish_gray_stack (int generation, GrayQueue *queue);
 552
 553 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
 554
 555
 556 static void init_stats (void);
 557
 558 static int mark_ephemerons_in_range (ScanCopyContext ctx);
 559 static void clear_unreachable_ephemerons (ScanCopyContext ctx);
 560 static void null_ephemerons_for_domain (MonoDomain *domain);
 561
 562 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
 563
 564 SgenObjectOperations current_object_ops;
 565 SgenMajorCollector major_collector;
 566 SgenMinorCollector sgen_minor_collector;
 567 static GrayQueue gray_queue;
 568
 569 static SgenRemeberedSet remset;
 570
 571 /* The gray queue to use from the main collection thread. */
 572 #define WORKERS_DISTRIBUTE_GRAY_QUEUE   (&gray_queue)
 573
 574 /*
 575  * The gray queue a worker job must use.  If we're not parallel or
 576  * concurrent, we use the main gray queue.
 577  */
 578 static SgenGrayQueue*
 579 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
 580 {
 581         return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
 582 }
 583
 584 static void
 585 gray_queue_redirect (SgenGrayQueue *queue)
 586 {
 587         gboolean wake = FALSE;
 588
 589
 590         for (;;) {
 591                 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
 592                 if (!section)
 593                         break;
 594                 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
 595                 wake = TRUE;
 596         }
 597
 598         if (wake) {
 599                 g_assert (concurrent_collection_in_progress ||
 600                                 (current_collection_generation == GENERATION_OLD && major_collector.is_parallel));
 601                 if (sgen_workers_have_started ()) {
 602                         sgen_workers_wake_up_all ();
 603                 } else {
 604                         if (concurrent_collection_in_progress)
 605                                 g_assert (current_collection_generation == -1);
 606                 }
 607         }
 608 }
 609
 610 static gboolean
 611 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
 612 {
 613         MonoObject *o = (MonoObject*)(obj);
 614         MonoObject *ref = (MonoObject*)*(ptr);
 615         int offset = (char*)(ptr) - (char*)o;
 616
 617         if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
 618                 return TRUE;
 619         if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
 620                 return TRUE;
 621
 622 #ifndef DISABLE_REMOTING
 623         if (mono_class_has_parent_fast (o->vtable->klass, mono_defaults.real_proxy_class) &&
 624                         offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
 625                 return TRUE;
 626 #endif
 627         /* Thread.cached_culture_info */
 628         if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
 629                         !strcmp (ref->vtable->klass->name, "CultureInfo") &&
 630                         !strcmp(o->vtable->klass->name_space, "System") &&
 631                         !strcmp(o->vtable->klass->name, "Object[]"))
 632                 return TRUE;
 633         /*
 634          *  at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
 635          * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
 636          * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
 637          * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
 638          * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
 639          * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
 640          * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
 641          * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
 642          * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
 643          */
 644         if (!strcmp (ref->vtable->klass->name_space, "System") &&
 645                         !strcmp (ref->vtable->klass->name, "Byte[]") &&
 646                         !strcmp (o->vtable->klass->name_space, "System.IO") &&
 647                         !strcmp (o->vtable->klass->name, "MemoryStream"))
 648                 return TRUE;
 649         /* append_job() in threadpool.c */
 650         if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
 651                         !strcmp (ref->vtable->klass->name, "AsyncResult") &&
 652                         !strcmp (o->vtable->klass->name_space, "System") &&
 653                         !strcmp (o->vtable->klass->name, "Object[]") &&
 654                         mono_thread_pool_is_queue_array ((MonoArray*) o))
 655                 return TRUE;
 656         return FALSE;
 657 }
 658
 659 static void
 660 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
 661 {
 662         MonoObject *o = (MonoObject*)(obj);
 663         MonoObject *ref = (MonoObject*)*(ptr);
 664         int offset = (char*)(ptr) - (char*)o;
 665         MonoClass *class;
 666         MonoClassField *field;
 667         char *str;
 668
 669         if (!ref || ref->vtable->domain == domain)
 670                 return;
 671         if (is_xdomain_ref_allowed (ptr, obj, domain))
 672                 return;
 673
 674         field = NULL;
 675         for (class = o->vtable->klass; class; class = class->parent) {
 676                 int i;
 677
 678                 for (i = 0; i < class->field.count; ++i) {
 679                         if (class->fields[i].offset == offset) {
 680                                 field = &class->fields[i];
 681                                 break;
 682                         }
 683                 }
 684                 if (field)
 685                         break;
 686         }
 687
 688         if (ref->vtable->klass == mono_defaults.string_class)
 689                 str = mono_string_to_utf8 ((MonoString*)ref);
 690         else
 691                 str = NULL;
 692         g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s)  -  pointed to by:\n",
 693                         o, o->vtable->klass->name_space, o->vtable->klass->name,
 694                         offset, field ? field->name : "",
 695                         ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
 696         mono_gc_scan_for_specific_ref (o, TRUE);
 697         if (str)
 698                 g_free (str);
 699 }
 700
 701 #undef HANDLE_PTR
 702 #define HANDLE_PTR(ptr,obj)     check_reference_for_xdomain ((ptr), (obj), domain)
 703
 704 static void
 705 scan_object_for_xdomain_refs (char *start, mword size, void *data)
 706 {
 707         MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
 708
 709         #include "sgen-scan-object.h"
 710 }
 711
 712 static gboolean scan_object_for_specific_ref_precise = TRUE;
 713
 714 #undef HANDLE_PTR
 715 #define HANDLE_PTR(ptr,obj) do {                \
 716         if ((MonoObject*)*(ptr) == key) {       \
 717         g_print ("found ref to %p in object %p (%s) at offset %td\n",   \
 718                         key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
 719         }                                                               \
 720         } while (0)
 721
 722 static void
 723 scan_object_for_specific_ref (char *start, MonoObject *key)
 724 {
 725         char *forwarded;
 726
 727         if ((forwarded = SGEN_OBJECT_IS_FORWARDED (start)))
 728                 start = forwarded;
 729
 730         if (scan_object_for_specific_ref_precise) {
 731                 #include "sgen-scan-object.h"
 732         } else {
 733                 mword *words = (mword*)start;
 734                 size_t size = safe_object_get_size ((MonoObject*)start);
 735                 int i;
 736                 for (i = 0; i < size / sizeof (mword); ++i) {
 737                         if (words [i] == (mword)key) {
 738                                 g_print ("found possible ref to %p in object %p (%s) at offset %td\n",
 739                                                 key, start, safe_name (start), i * sizeof (mword));
 740                         }
 741                 }
 742         }
 743 }
 744
 745 void
 746 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
 747 {
 748         while (start < end) {
 749                 size_t size;
 750                 char *obj;
 751
 752                 if (!*(void**)start) {
 753                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
 754                         continue;
 755                 }
 756
 757                 if (allow_flags) {
 758                         if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
 759                                 obj = start;
 760                 } else {
 761                         obj = start;
 762                 }
 763
 764                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
 765
 766                 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable)
 767                         callback (obj, size, data);
 768
 769                 start += size;
 770         }
 771 }
 772
 773 static void
 774 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
 775 {
 776         scan_object_for_specific_ref (obj, key);
 777 }
 778
 779 static void
 780 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
 781 {
 782         if (key != obj)
 783                 return;
 784         g_print ("found ref to %p in root record %p\n", key, root);
 785 }
 786
 787 static MonoObject *check_key = NULL;
 788 static RootRecord *check_root = NULL;
 789
 790 static void
 791 check_root_obj_specific_ref_from_marker (void **obj)
 792 {
 793         check_root_obj_specific_ref (check_root, check_key, *obj);
 794 }
 795
 796 static void
 797 scan_roots_for_specific_ref (MonoObject *key, int root_type)
 798 {
 799         void **start_root;
 800         RootRecord *root;
 801         check_key = key;
 802
 803         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 804                 mword desc = root->root_desc;
 805
 806                 check_root = root;
 807
 808                 switch (desc & ROOT_DESC_TYPE_MASK) {
 809                 case ROOT_DESC_BITMAP:
 810                         desc >>= ROOT_DESC_TYPE_SHIFT;
 811                         while (desc) {
 812                                 if (desc & 1)
 813                                         check_root_obj_specific_ref (root, key, *start_root);
 814                                 desc >>= 1;
 815                                 start_root++;
 816                         }
 817                         return;
 818                 case ROOT_DESC_COMPLEX: {
 819                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 820                         int bwords = (*bitmap_data) - 1;
 821                         void **start_run = start_root;
 822                         bitmap_data++;
 823                         while (bwords-- > 0) {
 824                                 gsize bmap = *bitmap_data++;
 825                                 void **objptr = start_run;
 826                                 while (bmap) {
 827                                         if (bmap & 1)
 828                                                 check_root_obj_specific_ref (root, key, *objptr);
 829                                         bmap >>= 1;
 830                                         ++objptr;
 831                                 }
 832                                 start_run += GC_BITS_PER_WORD;
 833                         }
 834                         break;
 835                 }
 836                 case ROOT_DESC_USER: {
 837                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 838                         marker (start_root, check_root_obj_specific_ref_from_marker);
 839                         break;
 840                 }
 841                 case ROOT_DESC_RUN_LEN:
 842                         g_assert_not_reached ();
 843                 default:
 844                         g_assert_not_reached ();
 845                 }
 846         } SGEN_HASH_TABLE_FOREACH_END;
 847
 848         check_key = NULL;
 849         check_root = NULL;
 850 }
 851
 852 void
 853 mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise)
 854 {
 855         void **ptr;
 856         RootRecord *root;
 857
 858         scan_object_for_specific_ref_precise = precise;
 859
 860         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 861                         (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key, TRUE);
 862
 863         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 864
 865         sgen_los_iterate_objects ((IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 866
 867         scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
 868         scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
 869
 870         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], ptr, root) {
 871                 while (ptr < (void**)root->end_root) {
 872                         check_root_obj_specific_ref (root, *ptr, key);
 873                         ++ptr;
 874                 }
 875         } SGEN_HASH_TABLE_FOREACH_END;
 876 }
 877
 878 static gboolean
 879 need_remove_object_for_domain (char *start, MonoDomain *domain)
 880 {
 881         if (mono_object_domain (start) == domain) {
 882                 SGEN_LOG (4, "Need to cleanup object %p", start);
 883                 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
 884                 return TRUE;
 885         }
 886         return FALSE;
 887 }
 888
 889 static void
 890 process_object_for_domain_clearing (char *start, MonoDomain *domain)
 891 {
 892         GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
 893         if (vt->klass == mono_defaults.internal_thread_class)
 894                 g_assert (mono_object_domain (start) == mono_get_root_domain ());
 895         /* The object could be a proxy for an object in the domain
 896            we're deleting. */
 897 #ifndef DISABLE_REMOTING
 898         if (mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
 899                 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
 900
 901                 /* The server could already have been zeroed out, so
 902                    we need to check for that, too. */
 903                 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
 904                         SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
 905                         ((MonoRealProxy*)start)->unwrapped_server = NULL;
 906                 }
 907         }
 908 #endif
 909 }
 910
 911 static MonoDomain *check_domain = NULL;
 912
 913 static void
 914 check_obj_not_in_domain (void **o)
 915 {
 916         g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
 917 }
 918
 919 static void
 920 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
 921 {
 922         void **start_root;
 923         RootRecord *root;
 924         check_domain = domain;
 925         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 926                 mword desc = root->root_desc;
 927
 928                 /* The MonoDomain struct is allowed to hold
 929                    references to objects in its own domain. */
 930                 if (start_root == (void**)domain)
 931                         continue;
 932
 933                 switch (desc & ROOT_DESC_TYPE_MASK) {
 934                 case ROOT_DESC_BITMAP:
 935                         desc >>= ROOT_DESC_TYPE_SHIFT;
 936                         while (desc) {
 937                                 if ((desc & 1) && *start_root)
 938                                         check_obj_not_in_domain (*start_root);
 939                                 desc >>= 1;
 940                                 start_root++;
 941                         }
 942                         break;
 943                 case ROOT_DESC_COMPLEX: {
 944                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 945                         int bwords = (*bitmap_data) - 1;
 946                         void **start_run = start_root;
 947                         bitmap_data++;
 948                         while (bwords-- > 0) {
 949                                 gsize bmap = *bitmap_data++;
 950                                 void **objptr = start_run;
 951                                 while (bmap) {
 952                                         if ((bmap & 1) && *objptr)
 953                                                 check_obj_not_in_domain (*objptr);
 954                                         bmap >>= 1;
 955                                         ++objptr;
 956                                 }
 957                                 start_run += GC_BITS_PER_WORD;
 958                         }
 959                         break;
 960                 }
 961                 case ROOT_DESC_USER: {
 962                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 963                         marker (start_root, check_obj_not_in_domain);
 964                         break;
 965                 }
 966                 case ROOT_DESC_RUN_LEN:
 967                         g_assert_not_reached ();
 968                 default:
 969                         g_assert_not_reached ();
 970                 }
 971         } SGEN_HASH_TABLE_FOREACH_END;
 972
 973         check_domain = NULL;
 974 }
 975
 976 static void
 977 check_for_xdomain_refs (void)
 978 {
 979         LOSObject *bigobj;
 980
 981         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 982                         (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL, FALSE);
 983
 984         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
 985
 986         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
 987                 scan_object_for_xdomain_refs (bigobj->data, sgen_los_object_size (bigobj), NULL);
 988 }
 989
 990 static gboolean
 991 clear_domain_process_object (char *obj, MonoDomain *domain)
 992 {
 993         gboolean remove;
 994
 995         process_object_for_domain_clearing (obj, domain);
 996         remove = need_remove_object_for_domain (obj, domain);
 997
 998         if (remove && ((MonoObject*)obj)->synchronisation) {
 999                 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1000                 if (dislink)
1001                         sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
1002         }
1003
1004         return remove;
1005 }
1006
1007 static void
1008 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1009 {
1010         if (clear_domain_process_object (obj, domain))
1011                 memset (obj, 0, size);
1012 }
1013
1014 static void
1015 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1016 {
1017         clear_domain_process_object (obj, domain);
1018 }
1019
1020 static void
1021 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1022 {
1023         if (need_remove_object_for_domain (obj, domain))
1024                 major_collector.free_non_pinned_object (obj, size);
1025 }
1026
1027 static void
1028 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1029 {
1030         if (need_remove_object_for_domain (obj, domain))
1031                 major_collector.free_pinned_object (obj, size);
1032 }
1033
1034 /*
1035  * When appdomains are unloaded we can easily remove objects that have finalizers,
1036  * but all the others could still be present in random places on the heap.
1037  * We need a sweep to get rid of them even though it's going to be costly
1038  * with big heaps.
1039  * The reason we need to remove them is because we access the vtable and class
1040  * structures to know the object size and the reference bitmap: once the domain is
1041  * unloaded the point to random memory.
1042  */
1043 void
1044 mono_gc_clear_domain (MonoDomain * domain)
1045 {
1046         LOSObject *bigobj, *prev;
1047         int i;
1048
1049         LOCK_GC;
1050
1051         if (concurrent_collection_in_progress)
1052                 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
1053         g_assert (!concurrent_collection_in_progress);
1054
1055         sgen_process_fin_stage_entries ();
1056         sgen_process_dislink_stage_entries ();
1057
1058         sgen_clear_nursery_fragments ();
1059
1060         if (xdomain_checks && domain != mono_get_root_domain ()) {
1061                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1062                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1063                 check_for_xdomain_refs ();
1064         }
1065
1066         /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1067         to memory returned to the OS.*/
1068         null_ephemerons_for_domain (domain);
1069
1070         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1071                 sgen_null_links_for_domain (domain, i);
1072
1073         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1074                 sgen_remove_finalizers_for_domain (domain, i);
1075
1076         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1077                         (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
1078
1079         /* We need two passes over major and large objects because
1080            freeing such objects might give their memory back to the OS
1081            (in the case of large objects) or obliterate its vtable
1082            (pinned objects with major-copying or pinned and non-pinned
1083            objects with major-mark&sweep), but we might need to
1084            dereference a pointer from an object to another object if
1085            the first object is a proxy. */
1086         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1087         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1088                 clear_domain_process_object (bigobj->data, domain);
1089
1090         prev = NULL;
1091         for (bigobj = los_object_list; bigobj;) {
1092                 if (need_remove_object_for_domain (bigobj->data, domain)) {
1093                         LOSObject *to_free = bigobj;
1094                         if (prev)
1095                                 prev->next = bigobj->next;
1096                         else
1097                                 los_object_list = bigobj->next;
1098                         bigobj = bigobj->next;
1099                         SGEN_LOG (4, "Freeing large object %p", bigobj->data);
1100                         sgen_los_free_object (to_free);
1101                         continue;
1102                 }
1103                 prev = bigobj;
1104                 bigobj = bigobj->next;
1105         }
1106         major_collector.iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1107         major_collector.iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1108
1109         if (domain == mono_get_root_domain ()) {
1110                 if (G_UNLIKELY (do_pin_stats))
1111                         sgen_pin_stats_print_class_stats ();
1112                 sgen_object_layout_dump (stdout);
1113         }
1114
1115         UNLOCK_GC;
1116 }
1117
1118 /*
1119  * sgen_add_to_global_remset:
1120  *
1121  *   The global remset contains locations which point into newspace after
1122  * a minor collection. This can happen if the objects they point to are pinned.
1123  *
1124  * LOCKING: If called from a parallel collector, the global remset
1125  * lock must be held.  For serial collectors that is not necessary.
1126  */
1127 void
1128 sgen_add_to_global_remset (gpointer ptr, gpointer obj)
1129 {
1130         SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
1131
1132         HEAVY_STAT (++stat_wbarrier_add_to_global_remset);
1133
1134         if (!major_collector.is_concurrent) {
1135                 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
1136         } else {
1137                 if (current_collection_generation == -1)
1138                         SGEN_ASSERT (5, sgen_concurrent_collection_in_progress (), "Global remsets outside of collection pauses can only be added by the concurrent collector");
1139         }
1140
1141         if (!object_is_pinned (obj))
1142                 SGEN_ASSERT (5, sgen_minor_collector.is_split || sgen_concurrent_collection_in_progress (), "Non-pinned objects can only remain in nursery if it is a split nursery");
1143         else if (sgen_cement_lookup_or_register (obj))
1144                 return;
1145
1146         remset.record_pointer (ptr);
1147
1148         if (G_UNLIKELY (do_pin_stats))
1149                 sgen_pin_stats_register_global_remset (obj);
1150
1151         SGEN_LOG (8, "Adding global remset for %p", ptr);
1152         binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
1153
1154
1155 #ifdef ENABLE_DTRACE
1156         if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
1157                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
1158                 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
1159                                 vt->klass->name_space, vt->klass->name);
1160         }
1161 #endif
1162 }
1163
1164 /*
1165  * sgen_drain_gray_stack:
1166  *
1167  *   Scan objects in the gray stack until the stack is empty. This should be called
1168  * frequently after each object is copied, to achieve better locality and cache
1169  * usage.
1170  */
1171 gboolean
1172 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
1173 {
1174         char *obj;
1175         ScanObjectFunc scan_func = ctx.scan_func;
1176         GrayQueue *queue = ctx.queue;
1177
1178         if (max_objs == -1) {
1179                 for (;;) {
1180                         GRAY_OBJECT_DEQUEUE (queue, obj);
1181                         if (!obj)
1182                                 return TRUE;
1183                         SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1184                         scan_func (obj, queue);
1185                 }
1186         } else {
1187                 int i;
1188
1189                 do {
1190                         for (i = 0; i != max_objs; ++i) {
1191                                 GRAY_OBJECT_DEQUEUE (queue, obj);
1192                                 if (!obj)
1193                                         return TRUE;
1194                                 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1195                                 scan_func (obj, queue);
1196                         }
1197                 } while (max_objs < 0);
1198                 return FALSE;
1199         }
1200 }
1201
1202 /*
1203  * Addresses from start to end are already sorted. This function finds
1204  * the object header for each address and pins the object. The
1205  * addresses must be inside the passed section.  The (start of the)
1206  * address array is overwritten with the addresses of the actually
1207  * pinned objects.  Return the number of pinned objects.
1208  */
1209 static int
1210 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx)
1211 {
1212         void *last = NULL;
1213         int count = 0;
1214         void *search_start;
1215         void *last_obj = NULL;
1216         size_t last_obj_size = 0;
1217         void *addr;
1218         int idx;
1219         void **definitely_pinned = start;
1220         ScanObjectFunc scan_func = ctx.scan_func;
1221         SgenGrayQueue *queue = ctx.queue;
1222
1223         sgen_nursery_allocator_prepare_for_pinning ();
1224
1225         while (start < end) {
1226                 addr = *start;
1227                 /* the range check should be reduntant */
1228                 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1229                         SGEN_LOG (5, "Considering pinning addr %p", addr);
1230                         /* multiple pointers to the same object */
1231                         if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1232                                 start++;
1233                                 continue;
1234                         }
1235                         idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1236                         g_assert (idx < section->num_scan_start);
1237                         search_start = (void*)section->scan_starts [idx];
1238                         if (!search_start || search_start > addr) {
1239                                 while (idx) {
1240                                         --idx;
1241                                         search_start = section->scan_starts [idx];
1242                                         if (search_start && search_start <= addr)
1243                                                 break;
1244                                 }
1245                                 if (!search_start || search_start > addr)
1246                                         search_start = start_nursery;
1247                         }
1248                         if (search_start < last_obj)
1249                                 search_start = (char*)last_obj + last_obj_size;
1250                         /* now addr should be in an object a short distance from search_start
1251                          * Note that search_start must point to zeroed mem or point to an object.
1252                          */
1253
1254                         do {
1255                                 if (!*(void**)search_start) {
1256                                         /* Consistency check */
1257                                         /*
1258                                         for (frag = nursery_fragments; frag; frag = frag->next) {
1259                                                 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1260                                                         g_assert_not_reached ();
1261                                         }
1262                                         */
1263
1264                                         search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
1265                                         continue;
1266                                 }
1267                                 last_obj = search_start;
1268                                 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1269
1270                                 if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) {
1271                                         /* Marks the beginning of a nursery fragment, skip */
1272                                 } else {
1273                                         SGEN_LOG (8, "Pinned try match %p (%s), size %zd", last_obj, safe_name (last_obj), last_obj_size);
1274                                         if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1275                                                 if (scan_func) {
1276                                                         scan_func (search_start, queue);
1277                                                 } else {
1278                                                         SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1279                                                                         search_start, *(void**)search_start, safe_name (search_start), count);
1280                                                         binary_protocol_pin (search_start,
1281                                                                         (gpointer)LOAD_VTABLE (search_start),
1282                                                                         safe_object_get_size (search_start));
1283
1284 #ifdef ENABLE_DTRACE
1285                                                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1286                                                                 int gen = sgen_ptr_in_nursery (search_start) ? GENERATION_NURSERY : GENERATION_OLD;
1287                                                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (search_start);
1288                                                                 MONO_GC_OBJ_PINNED ((mword)search_start,
1289                                                                                 sgen_safe_object_get_size (search_start),
1290                                                                                 vt->klass->name_space, vt->klass->name, gen);
1291                                                         }
1292 #endif
1293
1294                                                         pin_object (search_start);
1295                                                         GRAY_OBJECT_ENQUEUE (queue, search_start);
1296                                                         if (G_UNLIKELY (do_pin_stats))
1297                                                                 sgen_pin_stats_register_object (search_start, last_obj_size);
1298                                                         definitely_pinned [count] = search_start;
1299                                                         count++;
1300                                                 }
1301                                                 break;
1302                                         }
1303                                 }
1304                                 /* skip to the next object */
1305                                 search_start = (void*)((char*)search_start + last_obj_size);
1306                         } while (search_start <= addr);
1307                         /* we either pinned the correct object or we ignored the addr because
1308                          * it points to unused zeroed memory.
1309                          */
1310                         last = addr;
1311                 }
1312                 start++;
1313         }
1314         //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1315         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1316                 GCRootReport report;
1317                 report.count = 0;
1318                 for (idx = 0; idx < count; ++idx)
1319                         add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1320                 notify_gc_roots (&report);
1321         }
1322         stat_pinned_objects += count;
1323         return count;
1324 }
1325
1326 void
1327 sgen_pin_objects_in_section (GCMemSection *section, ScanCopyContext ctx)
1328 {
1329         int num_entries = section->pin_queue_num_entries;
1330         if (num_entries) {
1331                 void **start = section->pin_queue_start;
1332                 int reduced_to;
1333                 reduced_to = pin_objects_from_addresses (section, start, start + num_entries,
1334                                 section->data, section->next_data, ctx);
1335                 section->pin_queue_num_entries = reduced_to;
1336                 if (!reduced_to)
1337                         section->pin_queue_start = NULL;
1338         }
1339 }
1340
1341
1342 void
1343 sgen_pin_object (void *object, GrayQueue *queue)
1344 {
1345         g_assert (!concurrent_collection_in_progress);
1346
1347         if (sgen_collection_is_parallel ()) {
1348                 LOCK_PIN_QUEUE;
1349                 /*object arrives pinned*/
1350                 sgen_pin_stage_ptr (object);
1351                 ++objects_pinned ;
1352                 UNLOCK_PIN_QUEUE;
1353         } else {
1354                 SGEN_PIN_OBJECT (object);
1355                 sgen_pin_stage_ptr (object);
1356                 ++objects_pinned;
1357                 if (G_UNLIKELY (do_pin_stats))
1358                         sgen_pin_stats_register_object (object, safe_object_get_size (object));
1359         }
1360         GRAY_OBJECT_ENQUEUE (queue, object);
1361         binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1362
1363 #ifdef ENABLE_DTRACE
1364         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1365                 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1366                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1367                 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1368         }
1369 #endif
1370 }
1371
1372 void
1373 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1374 {
1375         for (;;) {
1376                 mword vtable_word;
1377                 gboolean major_pinned = FALSE;
1378
1379                 if (sgen_ptr_in_nursery (obj)) {
1380                         if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
1381                                 sgen_pin_object (obj, queue);
1382                                 break;
1383                         }
1384                 } else {
1385                         major_collector.pin_major_object (obj, queue);
1386                         major_pinned = TRUE;
1387                 }
1388
1389                 vtable_word = *(mword*)obj;
1390                 /*someone else forwarded it, update the pointer and bail out*/
1391                 if (vtable_word & SGEN_FORWARDED_BIT) {
1392                         *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
1393                         break;
1394                 }
1395
1396                 /*someone pinned it, nothing to do.*/
1397                 if (vtable_word & SGEN_PINNED_BIT || major_pinned)
1398                         break;
1399         }
1400 }
1401
1402 /* Sort the addresses in array in increasing order.
1403  * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1404  */
1405 void
1406 sgen_sort_addresses (void **array, int size)
1407 {
1408         int i;
1409         void *tmp;
1410
1411         for (i = 1; i < size; ++i) {
1412                 int child = i;
1413                 while (child > 0) {
1414                         int parent = (child - 1) / 2;
1415
1416                         if (array [parent] >= array [child])
1417                                 break;
1418
1419                         tmp = array [parent];
1420                         array [parent] = array [child];
1421                         array [child] = tmp;
1422
1423                         child = parent;
1424                 }
1425         }
1426
1427         for (i = size - 1; i > 0; --i) {
1428                 int end, root;
1429                 tmp = array [i];
1430                 array [i] = array [0];
1431                 array [0] = tmp;
1432
1433                 end = i - 1;
1434                 root = 0;
1435
1436                 while (root * 2 + 1 <= end) {
1437                         int child = root * 2 + 1;
1438
1439                         if (child < end && array [child] < array [child + 1])
1440                                 ++child;
1441                         if (array [root] >= array [child])
1442                                 break;
1443
1444                         tmp = array [root];
1445                         array [root] = array [child];
1446                         array [child] = tmp;
1447
1448                         root = child;
1449                 }
1450         }
1451 }
1452
1453 /*
1454  * Scan the memory between start and end and queue values which could be pointers
1455  * to the area between start_nursery and end_nursery for later consideration.
1456  * Typically used for thread stacks.
1457  */
1458 static void
1459 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1460 {
1461         int count = 0;
1462
1463 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1464         VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1465 #endif
1466
1467         while (start < end) {
1468                 if (*start >= start_nursery && *start < end_nursery) {
1469                         /*
1470                          * *start can point to the middle of an object
1471                          * note: should we handle pointing at the end of an object?
1472                          * pinning in C# code disallows pointing at the end of an object
1473                          * but there is some small chance that an optimizing C compiler
1474                          * may keep the only reference to an object by pointing
1475                          * at the end of it. We ignore this small chance for now.
1476                          * Pointers to the end of an object are indistinguishable
1477                          * from pointers to the start of the next object in memory
1478                          * so if we allow that we'd need to pin two objects...
1479                          * We queue the pointer in an array, the
1480                          * array will then be sorted and uniqued. This way
1481                          * we can coalesce several pinning pointers and it should
1482                          * be faster since we'd do a memory scan with increasing
1483                          * addresses. Note: we can align the address to the allocation
1484                          * alignment, so the unique process is more effective.
1485                          */
1486                         mword addr = (mword)*start;
1487                         addr &= ~(ALLOC_ALIGN - 1);
1488                         if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1489                                 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1490                                 sgen_pin_stage_ptr ((void*)addr);
1491                                 count++;
1492                         }
1493                         if (G_UNLIKELY (do_pin_stats)) {
1494                                 if (ptr_in_nursery ((void*)addr))
1495                                         sgen_pin_stats_register_address ((char*)addr, pin_type);
1496                         }
1497                 }
1498                 start++;
1499         }
1500         if (count)
1501                 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1502 }
1503
1504 /*
1505  * The first thing we do in a collection is to identify pinned objects.
1506  * This function considers all the areas of memory that need to be
1507  * conservatively scanned.
1508  */
1509 static void
1510 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1511 {
1512         void **start_root;
1513         RootRecord *root;
1514         SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1515         /* objects pinned from the API are inside these roots */
1516         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1517                 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1518                 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1519         } SGEN_HASH_TABLE_FOREACH_END;
1520         /* now deal with the thread stacks
1521          * in the future we should be able to conservatively scan only:
1522          * *) the cpu registers
1523          * *) the unmanaged stack frames
1524          * *) the _last_ managed stack frame
1525          * *) pointers slots in managed frames
1526          */
1527         scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1528 }
1529
1530 static void
1531 unpin_objects_from_queue (SgenGrayQueue *queue)
1532 {
1533         for (;;) {
1534                 char *addr;
1535                 GRAY_OBJECT_DEQUEUE (queue, addr);
1536                 if (!addr)
1537                         break;
1538                 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1539                 SGEN_UNPIN_OBJECT (addr);
1540         }
1541 }
1542
1543 typedef struct {
1544         CopyOrMarkObjectFunc func;
1545         GrayQueue *queue;
1546 } UserCopyOrMarkData;
1547
1548 static MonoNativeTlsKey user_copy_or_mark_key;
1549
1550 static void
1551 init_user_copy_or_mark_key (void)
1552 {
1553         mono_native_tls_alloc (&user_copy_or_mark_key, NULL);
1554 }
1555
1556 static void
1557 set_user_copy_or_mark_data (UserCopyOrMarkData *data)
1558 {
1559         mono_native_tls_set_value (user_copy_or_mark_key, data);
1560 }
1561
1562 static void
1563 single_arg_user_copy_or_mark (void **obj)
1564 {
1565         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
1566
1567         data->func (obj, data->queue);
1568 }
1569
1570 /*
1571  * The memory area from start_root to end_root contains pointers to objects.
1572  * Their position is precisely described by @desc (this means that the pointer
1573  * can be either NULL or the pointer to the start of an object).
1574  * This functions copies them to to_space updates them.
1575  *
1576  * This function is not thread-safe!
1577  */
1578 static void
1579 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1580 {
1581         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1582         SgenGrayQueue *queue = ctx.queue;
1583
1584         switch (desc & ROOT_DESC_TYPE_MASK) {
1585         case ROOT_DESC_BITMAP:
1586                 desc >>= ROOT_DESC_TYPE_SHIFT;
1587                 while (desc) {
1588                         if ((desc & 1) && *start_root) {
1589                                 copy_func (start_root, queue);
1590                                 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1591                                 sgen_drain_gray_stack (-1, ctx);
1592                         }
1593                         desc >>= 1;
1594                         start_root++;
1595                 }
1596                 return;
1597         case ROOT_DESC_COMPLEX: {
1598                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1599                 int bwords = (*bitmap_data) - 1;
1600                 void **start_run = start_root;
1601                 bitmap_data++;
1602                 while (bwords-- > 0) {
1603                         gsize bmap = *bitmap_data++;
1604                         void **objptr = start_run;
1605                         while (bmap) {
1606                                 if ((bmap & 1) && *objptr) {
1607                                         copy_func (objptr, queue);
1608                                         SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1609                                         sgen_drain_gray_stack (-1, ctx);
1610                                 }
1611                                 bmap >>= 1;
1612                                 ++objptr;
1613                         }
1614                         start_run += GC_BITS_PER_WORD;
1615                 }
1616                 break;
1617         }
1618         case ROOT_DESC_USER: {
1619                 UserCopyOrMarkData data = { copy_func, queue };
1620                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1621                 set_user_copy_or_mark_data (&data);
1622                 marker (start_root, single_arg_user_copy_or_mark);
1623                 set_user_copy_or_mark_data (NULL);
1624                 break;
1625         }
1626         case ROOT_DESC_RUN_LEN:
1627                 g_assert_not_reached ();
1628         default:
1629                 g_assert_not_reached ();
1630         }
1631 }
1632
1633 static void
1634 reset_heap_boundaries (void)
1635 {
1636         lowest_heap_address = ~(mword)0;
1637         highest_heap_address = 0;
1638 }
1639
1640 void
1641 sgen_update_heap_boundaries (mword low, mword high)
1642 {
1643         mword old;
1644
1645         do {
1646                 old = lowest_heap_address;
1647                 if (low >= old)
1648                         break;
1649         } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1650
1651         do {
1652                 old = highest_heap_address;
1653                 if (high <= old)
1654                         break;
1655         } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1656 }
1657
1658 /*
1659  * Allocate and setup the data structures needed to be able to allocate objects
1660  * in the nursery. The nursery is stored in nursery_section.
1661  */
1662 static void
1663 alloc_nursery (void)
1664 {
1665         GCMemSection *section;
1666         char *data;
1667         int scan_starts;
1668         int alloc_size;
1669
1670         if (nursery_section)
1671                 return;
1672         SGEN_LOG (2, "Allocating nursery size: %lu", (unsigned long)sgen_nursery_size);
1673         /* later we will alloc a larger area for the nursery but only activate
1674          * what we need. The rest will be used as expansion if we have too many pinned
1675          * objects in the existing nursery.
1676          */
1677         /* FIXME: handle OOM */
1678         section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1679
1680         alloc_size = sgen_nursery_size;
1681
1682         /* If there isn't enough space even for the nursery we should simply abort. */
1683         g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1684
1685 #ifdef SGEN_ALIGN_NURSERY
1686         data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1687 #else
1688         data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1689 #endif
1690         sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1691         SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1692         section->data = section->next_data = data;
1693         section->size = alloc_size;
1694         section->end_data = data + sgen_nursery_size;
1695         scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1696         section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1697         section->num_scan_start = scan_starts;
1698
1699         nursery_section = section;
1700
1701         sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1702 }
1703
1704 void*
1705 mono_gc_get_nursery (int *shift_bits, size_t *size)
1706 {
1707         *size = sgen_nursery_size;
1708 #ifdef SGEN_ALIGN_NURSERY
1709         *shift_bits = DEFAULT_NURSERY_BITS;
1710 #else
1711         *shift_bits = -1;
1712 #endif
1713         return sgen_get_nursery_start ();
1714 }
1715
1716 void
1717 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1718 {
1719         SgenThreadInfo *info = mono_thread_info_current ();
1720
1721         /* Could be called from sgen_thread_unregister () with a NULL info */
1722         if (domain) {
1723                 g_assert (info);
1724                 info->stopped_domain = domain;
1725         }
1726 }
1727
1728 gboolean
1729 mono_gc_precise_stack_mark_enabled (void)
1730 {
1731         return !conservative_stack_mark;
1732 }
1733
1734 FILE *
1735 mono_gc_get_logfile (void)
1736 {
1737         return gc_debug_file;
1738 }
1739
1740 static void
1741 report_finalizer_roots_list (FinalizeReadyEntry *list)
1742 {
1743         GCRootReport report;
1744         FinalizeReadyEntry *fin;
1745
1746         report.count = 0;
1747         for (fin = list; fin; fin = fin->next) {
1748                 if (!fin->object)
1749                         continue;
1750                 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1751         }
1752         notify_gc_roots (&report);
1753 }
1754
1755 static void
1756 report_finalizer_roots (void)
1757 {
1758         report_finalizer_roots_list (fin_ready_list);
1759         report_finalizer_roots_list (critical_fin_list);
1760 }
1761
1762 static GCRootReport *root_report;
1763
1764 static void
1765 single_arg_report_root (void **obj)
1766 {
1767         if (*obj)
1768                 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1769 }
1770
1771 static void
1772 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1773 {
1774         switch (desc & ROOT_DESC_TYPE_MASK) {
1775         case ROOT_DESC_BITMAP:
1776                 desc >>= ROOT_DESC_TYPE_SHIFT;
1777                 while (desc) {
1778                         if ((desc & 1) && *start_root) {
1779                                 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1780                         }
1781                         desc >>= 1;
1782                         start_root++;
1783                 }
1784                 return;
1785         case ROOT_DESC_COMPLEX: {
1786                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1787                 int bwords = (*bitmap_data) - 1;
1788                 void **start_run = start_root;
1789                 bitmap_data++;
1790                 while (bwords-- > 0) {
1791                         gsize bmap = *bitmap_data++;
1792                         void **objptr = start_run;
1793                         while (bmap) {
1794                                 if ((bmap & 1) && *objptr) {
1795                                         add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1796                                 }
1797                                 bmap >>= 1;
1798                                 ++objptr;
1799                         }
1800                         start_run += GC_BITS_PER_WORD;
1801                 }
1802                 break;
1803         }
1804         case ROOT_DESC_USER: {
1805                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1806                 root_report = report;
1807                 marker (start_root, single_arg_report_root);
1808                 break;
1809         }
1810         case ROOT_DESC_RUN_LEN:
1811                 g_assert_not_reached ();
1812         default:
1813                 g_assert_not_reached ();
1814         }
1815 }
1816
1817 static void
1818 report_registered_roots_by_type (int root_type)
1819 {
1820         GCRootReport report;
1821         void **start_root;
1822         RootRecord *root;
1823         report.count = 0;
1824         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1825                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1826                 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1827         } SGEN_HASH_TABLE_FOREACH_END;
1828         notify_gc_roots (&report);
1829 }
1830
1831 static void
1832 report_registered_roots (void)
1833 {
1834         report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1835         report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1836 }
1837
1838 static void
1839 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1840 {
1841         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1842         SgenGrayQueue *queue = ctx.queue;
1843         FinalizeReadyEntry *fin;
1844
1845         for (fin = list; fin; fin = fin->next) {
1846                 if (!fin->object)
1847                         continue;
1848                 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1849                 copy_func (&fin->object, queue);
1850         }
1851 }
1852
1853 static const char*
1854 generation_name (int generation)
1855 {
1856         switch (generation) {
1857         case GENERATION_NURSERY: return "nursery";
1858         case GENERATION_OLD: return "old";
1859         default: g_assert_not_reached ();
1860         }
1861 }
1862
1863 const char*
1864 sgen_generation_name (int generation)
1865 {
1866         return generation_name (generation);
1867 }
1868
1869 SgenObjectOperations *
1870 sgen_get_current_object_ops (void){
1871         return &current_object_ops;
1872 }
1873
1874
1875 static void
1876 finish_gray_stack (int generation, GrayQueue *queue)
1877 {
1878         TV_DECLARE (atv);
1879         TV_DECLARE (btv);
1880         int done_with_ephemerons, ephemeron_rounds = 0;
1881         CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1882         ScanObjectFunc scan_func = current_object_ops.scan_object;
1883         ScanCopyContext ctx = { scan_func, copy_func, queue };
1884         char *start_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_start () : NULL;
1885         char *end_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_end () : (char*)-1;
1886
1887         /*
1888          * We copied all the reachable objects. Now it's the time to copy
1889          * the objects that were not referenced by the roots, but by the copied objects.
1890          * we built a stack of objects pointed to by gray_start: they are
1891          * additional roots and we may add more items as we go.
1892          * We loop until gray_start == gray_objects which means no more objects have
1893          * been added. Note this is iterative: no recursion is involved.
1894          * We need to walk the LO list as well in search of marked big objects
1895          * (use a flag since this is needed only on major collections). We need to loop
1896          * here as well, so keep a counter of marked LO (increasing it in copy_object).
1897          *   To achieve better cache locality and cache usage, we drain the gray stack
1898          * frequently, after each object is copied, and just finish the work here.
1899          */
1900         sgen_drain_gray_stack (-1, ctx);
1901         TV_GETTIME (atv);
1902         SGEN_LOG (2, "%s generation done", generation_name (generation));
1903
1904         /*
1905         Reset bridge data, we might have lingering data from a previous collection if this is a major
1906         collection trigged by minor overflow.
1907
1908         We must reset the gathered bridges since their original block might be evacuated due to major
1909         fragmentation in the meanwhile and the bridge code should not have to deal with that.
1910         */
1911         sgen_bridge_reset_data ();
1912
1913         /*
1914          * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1915          * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1916          * objects that are in fact reachable.
1917          */
1918         done_with_ephemerons = 0;
1919         do {
1920                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1921                 sgen_drain_gray_stack (-1, ctx);
1922                 ++ephemeron_rounds;
1923         } while (!done_with_ephemerons);
1924
1925         sgen_scan_togglerefs (start_addr, end_addr, ctx);
1926
1927         if (sgen_need_bridge_processing ()) {
1928                 sgen_collect_bridge_objects (generation, ctx);
1929                 if (generation == GENERATION_OLD)
1930                         sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1931         }
1932
1933         /*
1934         Make sure we drain the gray stack before processing disappearing links and finalizers.
1935         If we don't make sure it is empty we might wrongly see a live object as dead.
1936         */
1937         sgen_drain_gray_stack (-1, ctx);
1938
1939         /*
1940         We must clear weak links that don't track resurrection before processing object ready for
1941         finalization so they can be cleared before that.
1942         */
1943         sgen_null_link_in_range (generation, TRUE, ctx);
1944         if (generation == GENERATION_OLD)
1945                 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1946
1947
1948         /* walk the finalization queue and move also the objects that need to be
1949          * finalized: use the finalized objects as new roots so the objects they depend
1950          * on are also not reclaimed. As with the roots above, only objects in the nursery
1951          * are marked/copied.
1952          */
1953         sgen_finalize_in_range (generation, ctx);
1954         if (generation == GENERATION_OLD)
1955                 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1956         /* drain the new stack that might have been created */
1957         SGEN_LOG (6, "Precise scan of gray area post fin");
1958         sgen_drain_gray_stack (-1, ctx);
1959
1960         /*
1961          * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1962          */
1963         done_with_ephemerons = 0;
1964         do {
1965                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1966                 sgen_drain_gray_stack (-1, ctx);
1967                 ++ephemeron_rounds;
1968         } while (!done_with_ephemerons);
1969
1970         /*
1971          * Clear ephemeron pairs with unreachable keys.
1972          * We pass the copy func so we can figure out if an array was promoted or not.
1973          */
1974         clear_unreachable_ephemerons (ctx);
1975
1976         TV_GETTIME (btv);
1977         SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
1978
1979         /*
1980          * handle disappearing links
1981          * Note we do this after checking the finalization queue because if an object
1982          * survives (at least long enough to be finalized) we don't clear the link.
1983          * This also deals with a possible issue with the monitor reclamation: with the Boehm
1984          * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1985          * called.
1986          */
1987         g_assert (sgen_gray_object_queue_is_empty (queue));
1988         for (;;) {
1989                 sgen_null_link_in_range (generation, FALSE, ctx);
1990                 if (generation == GENERATION_OLD)
1991                         sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
1992                 if (sgen_gray_object_queue_is_empty (queue))
1993                         break;
1994                 sgen_drain_gray_stack (-1, ctx);
1995         }
1996
1997         g_assert (sgen_gray_object_queue_is_empty (queue));
1998 }
1999
2000 void
2001 sgen_check_section_scan_starts (GCMemSection *section)
2002 {
2003         int i;
2004         for (i = 0; i < section->num_scan_start; ++i) {
2005                 if (section->scan_starts [i]) {
2006                         guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2007                         g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2008                 }
2009         }
2010 }
2011
2012 static void
2013 check_scan_starts (void)
2014 {
2015         if (!do_scan_starts_check)
2016                 return;
2017         sgen_check_section_scan_starts (nursery_section);
2018         major_collector.check_scan_starts ();
2019 }
2020
2021 static void
2022 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
2023 {
2024         void **start_root;
2025         RootRecord *root;
2026         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
2027                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
2028                 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
2029         } SGEN_HASH_TABLE_FOREACH_END;
2030 }
2031
2032 void
2033 sgen_dump_occupied (char *start, char *end, char *section_start)
2034 {
2035         fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2036 }
2037
2038 void
2039 sgen_dump_section (GCMemSection *section, const char *type)
2040 {
2041         char *start = section->data;
2042         char *end = section->data + section->size;
2043         char *occ_start = NULL;
2044         GCVTable *vt;
2045         char *old_start = NULL; /* just for debugging */
2046
2047         fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2048
2049         while (start < end) {
2050                 guint size;
2051                 MonoClass *class;
2052
2053                 if (!*(void**)start) {
2054                         if (occ_start) {
2055                                 sgen_dump_occupied (occ_start, start, section->data);
2056                                 occ_start = NULL;
2057                         }
2058                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2059                         continue;
2060                 }
2061                 g_assert (start < section->next_data);
2062
2063                 if (!occ_start)
2064                         occ_start = start;
2065
2066                 vt = (GCVTable*)LOAD_VTABLE (start);
2067                 class = vt->klass;
2068
2069                 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2070
2071                 /*
2072                 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2073                                 start - section->data,
2074                                 vt->klass->name_space, vt->klass->name,
2075                                 size);
2076                 */
2077
2078                 old_start = start;
2079                 start += size;
2080         }
2081         if (occ_start)
2082                 sgen_dump_occupied (occ_start, start, section->data);
2083
2084         fprintf (heap_dump_file, "</section>\n");
2085 }
2086
2087 static void
2088 dump_object (MonoObject *obj, gboolean dump_location)
2089 {
2090         static char class_name [1024];
2091
2092         MonoClass *class = mono_object_class (obj);
2093         int i, j;
2094
2095         /*
2096          * Python's XML parser is too stupid to parse angle brackets
2097          * in strings, so we just ignore them;
2098          */
2099         i = j = 0;
2100         while (class->name [i] && j < sizeof (class_name) - 1) {
2101                 if (!strchr ("<>\"", class->name [i]))
2102                         class_name [j++] = class->name [i];
2103                 ++i;
2104         }
2105         g_assert (j < sizeof (class_name));
2106         class_name [j] = 0;
2107
2108         fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2109                         class->name_space, class_name,
2110                         safe_object_get_size (obj));
2111         if (dump_location) {
2112                 const char *location;
2113                 if (ptr_in_nursery (obj))
2114                         location = "nursery";
2115                 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2116                         location = "major";
2117                 else
2118                         location = "LOS";
2119                 fprintf (heap_dump_file, " location=\"%s\"", location);
2120         }
2121         fprintf (heap_dump_file, "/>\n");
2122 }
2123
2124 static void
2125 dump_heap (const char *type, int num, const char *reason)
2126 {
2127         ObjectList *list;
2128         LOSObject *bigobj;
2129
2130         fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2131         if (reason)
2132                 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2133         fprintf (heap_dump_file, ">\n");
2134         fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2135         sgen_dump_internal_mem_usage (heap_dump_file);
2136         fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
2137         /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2138         fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
2139
2140         fprintf (heap_dump_file, "<pinned-objects>\n");
2141         for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
2142                 dump_object (list->obj, TRUE);
2143         fprintf (heap_dump_file, "</pinned-objects>\n");
2144
2145         sgen_dump_section (nursery_section, "nursery");
2146
2147         major_collector.dump_heap (heap_dump_file);
2148
2149         fprintf (heap_dump_file, "<los>\n");
2150         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2151                 dump_object ((MonoObject*)bigobj->data, FALSE);
2152         fprintf (heap_dump_file, "</los>\n");
2153
2154         fprintf (heap_dump_file, "</collection>\n");
2155 }
2156
2157 void
2158 sgen_register_moved_object (void *obj, void *destination)
2159 {
2160         g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
2161
2162         /* FIXME: handle this for parallel collector */
2163         g_assert (!sgen_collection_is_parallel ());
2164
2165         if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2166                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2167                 moved_objects_idx = 0;
2168         }
2169         moved_objects [moved_objects_idx++] = obj;
2170         moved_objects [moved_objects_idx++] = destination;
2171 }
2172
2173 static void
2174 init_stats (void)
2175 {
2176         static gboolean inited = FALSE;
2177
2178         if (inited)
2179                 return;
2180
2181         mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pre_collection_fragment_clear);
2182         mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pinning);
2183         mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_remsets);
2184         mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_pinned);
2185         mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_registered_roots);
2186         mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_thread_data);
2187         mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_finish_gray_stack);
2188         mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_fragment_creation);
2189
2190         mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pre_collection_fragment_clear);
2191         mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pinning);
2192         mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_pinned);
2193         mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_registered_roots);
2194         mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_thread_data);
2195         mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_alloc_pinned);
2196         mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_finalized);
2197         mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_big_objects);
2198         mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_finish_gray_stack);
2199         mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_free_bigobjs);
2200         mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_los_sweep);
2201         mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_sweep);
2202         mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_fragment_creation);
2203
2204         mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_pinned_objects);
2205
2206 #ifdef HEAVY_STATISTICS
2207         mono_counters_register ("WBarrier remember pointer", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_add_to_global_remset);
2208         mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
2209         mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
2210         mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
2211         mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
2212         mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
2213         mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
2214         mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
2215
2216         mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
2217         mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
2218
2219         mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
2220         mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
2221         mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
2222         mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
2223
2224         mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
2225         mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
2226
2227         mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
2228
2229         mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
2230         mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
2231         mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
2232         mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_to_space);
2233
2234         sgen_nursery_allocator_init_heavy_stats ();
2235         sgen_alloc_init_heavy_stats ();
2236 #endif
2237
2238         inited = TRUE;
2239 }
2240
2241
2242 static void
2243 reset_pinned_from_failed_allocation (void)
2244 {
2245         bytes_pinned_from_failed_allocation = 0;
2246 }
2247
2248 void
2249 sgen_set_pinned_from_failed_allocation (mword objsize)
2250 {
2251         bytes_pinned_from_failed_allocation += objsize;
2252 }
2253
2254 gboolean
2255 sgen_collection_is_parallel (void)
2256 {
2257         switch (current_collection_generation) {
2258         case GENERATION_NURSERY:
2259                 return nursery_collection_is_parallel;
2260         case GENERATION_OLD:
2261                 return major_collector.is_parallel;
2262         default:
2263                 g_error ("Invalid current generation %d", current_collection_generation);
2264         }
2265 }
2266
2267 gboolean
2268 sgen_collection_is_concurrent (void)
2269 {
2270         switch (current_collection_generation) {
2271         case GENERATION_NURSERY:
2272                 return FALSE;
2273         case GENERATION_OLD:
2274                 return concurrent_collection_in_progress;
2275         default:
2276                 g_error ("Invalid current generation %d", current_collection_generation);
2277         }
2278 }
2279
2280 gboolean
2281 sgen_concurrent_collection_in_progress (void)
2282 {
2283         return concurrent_collection_in_progress;
2284 }
2285
2286 typedef struct
2287 {
2288         char *heap_start;
2289         char *heap_end;
2290 } FinishRememberedSetScanJobData;
2291
2292 static void
2293 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2294 {
2295         FinishRememberedSetScanJobData *job_data = job_data_untyped;
2296
2297         remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2298         sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2299 }
2300
2301 typedef struct
2302 {
2303         CopyOrMarkObjectFunc copy_or_mark_func;
2304         ScanObjectFunc scan_func;
2305         char *heap_start;
2306         char *heap_end;
2307         int root_type;
2308 } ScanFromRegisteredRootsJobData;
2309
2310 static void
2311 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2312 {
2313         ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2314         ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2315                 sgen_workers_get_job_gray_queue (worker_data) };
2316
2317         scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2318         sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2319 }
2320
2321 typedef struct
2322 {
2323         char *heap_start;
2324         char *heap_end;
2325 } ScanThreadDataJobData;
2326
2327 static void
2328 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2329 {
2330         ScanThreadDataJobData *job_data = job_data_untyped;
2331
2332         scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2333                         sgen_workers_get_job_gray_queue (worker_data));
2334         sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2335 }
2336
2337 typedef struct
2338 {
2339         FinalizeReadyEntry *list;
2340 } ScanFinalizerEntriesJobData;
2341
2342 static void
2343 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2344 {
2345         ScanFinalizerEntriesJobData *job_data = job_data_untyped;
2346         ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2347
2348         scan_finalizer_entries (job_data->list, ctx);
2349         sgen_free_internal_dynamic (job_data, sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2350 }
2351
2352 static void
2353 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2354 {
2355         g_assert (concurrent_collection_in_progress);
2356         major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2357 }
2358
2359 static void
2360 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2361 {
2362         g_assert (concurrent_collection_in_progress);
2363         sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2364 }
2365
2366 static void
2367 verify_scan_starts (char *start, char *end)
2368 {
2369         int i;
2370
2371         for (i = 0; i < nursery_section->num_scan_start; ++i) {
2372                 char *addr = nursery_section->scan_starts [i];
2373                 if (addr > start && addr < end)
2374                         SGEN_LOG (1, "NFC-BAD SCAN START [%d] %p for obj [%p %p]", i, addr, start, end);
2375         }
2376 }
2377
2378 static void
2379 verify_nursery (void)
2380 {
2381         char *start, *end, *cur, *hole_start;
2382
2383         if (!do_verify_nursery)
2384                 return;
2385
2386         /*This cleans up unused fragments */
2387         sgen_nursery_allocator_prepare_for_pinning ();
2388
2389         hole_start = start = cur = sgen_get_nursery_start ();
2390         end = sgen_get_nursery_end ();
2391
2392         while (cur < end) {
2393                 size_t ss, size;
2394
2395                 if (!*(void**)cur) {
2396                         cur += sizeof (void*);
2397                         continue;
2398                 }
2399
2400                 if (object_is_forwarded (cur))
2401                         SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2402                 else if (object_is_pinned (cur))
2403                         SGEN_LOG (1, "PINNED OBJ %p", cur);
2404
2405                 ss = safe_object_get_size ((MonoObject*)cur);
2406                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2407                 verify_scan_starts (cur, cur + size);
2408                 if (do_dump_nursery_content) {
2409                         if (cur > hole_start)
2410                                 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2411                         SGEN_LOG (1, "OBJ  [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2412                 }
2413                 cur += size;
2414                 hole_start = cur;
2415         }
2416 }
2417
2418 /*
2419  * Checks that no objects in the nursery are fowarded or pinned.  This
2420  * is a precondition to restarting the mutator while doing a
2421  * concurrent collection.  Note that we don't clear fragments because
2422  * we depend on that having happened earlier.
2423  */
2424 static void
2425 check_nursery_is_clean (void)
2426 {
2427         char *start, *end, *cur;
2428
2429         start = cur = sgen_get_nursery_start ();
2430         end = sgen_get_nursery_end ();
2431
2432         while (cur < end) {
2433                 size_t ss, size;
2434
2435                 if (!*(void**)cur) {
2436                         cur += sizeof (void*);
2437                         continue;
2438                 }
2439
2440                 g_assert (!object_is_forwarded (cur));
2441                 g_assert (!object_is_pinned (cur));
2442
2443                 ss = safe_object_get_size ((MonoObject*)cur);
2444                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2445                 verify_scan_starts (cur, cur + size);
2446
2447                 cur += size;
2448         }
2449 }
2450
2451 static void
2452 init_gray_queue (void)
2453 {
2454         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) {
2455                 sgen_workers_init_distribute_gray_queue ();
2456                 sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
2457                                 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2458         } else {
2459                 sgen_gray_object_queue_init (&gray_queue, NULL);
2460         }
2461 }
2462
2463 static void
2464 pin_stage_object_callback (char *obj, size_t size, void *data)
2465 {
2466         sgen_pin_stage_ptr (obj);
2467         /* FIXME: do pin stats if enabled */
2468 }
2469
2470 /*
2471  * Collect objects in the nursery.  Returns whether to trigger a major
2472  * collection.
2473  */
2474 static gboolean
2475 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2476 {
2477         gboolean needs_major;
2478         size_t max_garbage_amount;
2479         char *nursery_next;
2480         FinishRememberedSetScanJobData *frssjd;
2481         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2482         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2483         ScanThreadDataJobData *stdjd;
2484         mword fragment_total;
2485         ScanCopyContext ctx;
2486         TV_DECLARE (all_atv);
2487         TV_DECLARE (all_btv);
2488         TV_DECLARE (atv);
2489         TV_DECLARE (btv);
2490
2491         if (disable_minor_collections)
2492                 return TRUE;
2493
2494         MONO_GC_BEGIN (GENERATION_NURSERY);
2495         binary_protocol_collection_begin (stat_minor_gcs, GENERATION_NURSERY);
2496
2497         verify_nursery ();
2498
2499 #ifndef DISABLE_PERFCOUNTERS
2500         mono_perfcounters->gc_collections0++;
2501 #endif
2502
2503         current_collection_generation = GENERATION_NURSERY;
2504         if (sgen_collection_is_parallel ())
2505                 current_object_ops = sgen_minor_collector.parallel_ops;
2506         else
2507                 current_object_ops = sgen_minor_collector.serial_ops;
2508
2509         reset_pinned_from_failed_allocation ();
2510
2511         check_scan_starts ();
2512
2513         sgen_nursery_alloc_prepare_for_minor ();
2514
2515         degraded_mode = 0;
2516         objects_pinned = 0;
2517         nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2518         /* FIXME: optimize later to use the higher address where an object can be present */
2519         nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2520
2521         SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", stat_minor_gcs, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2522         max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2523         g_assert (nursery_section->size >= max_garbage_amount);
2524
2525         /* world must be stopped already */
2526         TV_GETTIME (all_atv);
2527         atv = all_atv;
2528
2529         TV_GETTIME (btv);
2530         time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2531
2532         if (xdomain_checks) {
2533                 sgen_clear_nursery_fragments ();
2534                 check_for_xdomain_refs ();
2535         }
2536
2537         nursery_section->next_data = nursery_next;
2538
2539         major_collector.start_nursery_collection ();
2540
2541         sgen_memgov_minor_collection_start ();
2542
2543         init_gray_queue ();
2544
2545         stat_minor_gcs++;
2546         gc_stats.minor_gc_count ++;
2547
2548         MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2549
2550         sgen_process_fin_stage_entries ();
2551         sgen_process_dislink_stage_entries ();
2552
2553         MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2554
2555         /* pin from pinned handles */
2556         sgen_init_pinning ();
2557         mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2558         pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2559         /* pin cemented objects */
2560         sgen_cement_iterate (pin_stage_object_callback, NULL);
2561         /* identify pinned objects */
2562         sgen_optimize_pin_queue (0);
2563         sgen_pinning_setup_section (nursery_section);
2564         ctx.scan_func = NULL;
2565         ctx.copy_func = NULL;
2566         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2567         sgen_pin_objects_in_section (nursery_section, ctx);
2568         sgen_pinning_trim_queue_to_section (nursery_section);
2569
2570         TV_GETTIME (atv);
2571         time_minor_pinning += TV_ELAPSED (btv, atv);
2572         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2573         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2574
2575         MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2576
2577         if (whole_heap_check_before_collection) {
2578                 sgen_clear_nursery_fragments ();
2579                 sgen_check_whole_heap (finish_up_concurrent_mark);
2580         }
2581         if (consistency_check_at_minor_collection)
2582                 sgen_check_consistency ();
2583
2584         sgen_workers_start_all_workers ();
2585         sgen_workers_start_marking ();
2586
2587         frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2588         frssjd->heap_start = sgen_get_nursery_start ();
2589         frssjd->heap_end = nursery_next;
2590         sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2591
2592         /* we don't have complete write barrier yet, so we scan all the old generation sections */
2593         TV_GETTIME (btv);
2594         time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2595         SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2596
2597         MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2598
2599         if (!sgen_collection_is_parallel ()) {
2600                 ctx.scan_func = current_object_ops.scan_object;
2601                 ctx.copy_func = NULL;
2602                 ctx.queue = &gray_queue;
2603                 sgen_drain_gray_stack (-1, ctx);
2604         }
2605
2606         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2607                 report_registered_roots ();
2608         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2609                 report_finalizer_roots ();
2610         TV_GETTIME (atv);
2611         time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2612
2613         MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2614
2615         /* registered roots, this includes static fields */
2616         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2617         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2618         scrrjd_normal->scan_func = current_object_ops.scan_object;
2619         scrrjd_normal->heap_start = sgen_get_nursery_start ();
2620         scrrjd_normal->heap_end = nursery_next;
2621         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2622         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2623
2624         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2625         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2626         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2627         scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2628         scrrjd_wbarrier->heap_end = nursery_next;
2629         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2630         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2631
2632         TV_GETTIME (btv);
2633         time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2634
2635         MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2636
2637         /* thread data */
2638         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2639         stdjd->heap_start = sgen_get_nursery_start ();
2640         stdjd->heap_end = nursery_next;
2641         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2642
2643         TV_GETTIME (atv);
2644         time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2645         btv = atv;
2646
2647         MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2648
2649         g_assert (!sgen_collection_is_parallel () && !sgen_collection_is_concurrent ());
2650
2651         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ())
2652                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2653
2654         /* Scan the list of objects ready for finalization. If */
2655         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2656         sfejd_fin_ready->list = fin_ready_list;
2657         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
2658
2659         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2660         sfejd_critical_fin->list = critical_fin_list;
2661         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
2662
2663         MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2664
2665         finish_gray_stack (GENERATION_NURSERY, &gray_queue);
2666         TV_GETTIME (atv);
2667         time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2668         mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2669
2670         MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2671
2672         /*
2673          * The (single-threaded) finalization code might have done
2674          * some copying/marking so we can only reset the GC thread's
2675          * worker data here instead of earlier when we joined the
2676          * workers.
2677          */
2678         sgen_workers_reset_data ();
2679
2680         if (objects_pinned) {
2681                 sgen_optimize_pin_queue (0);
2682                 sgen_pinning_setup_section (nursery_section);
2683         }
2684
2685         /* walk the pin_queue, build up the fragment list of free memory, unmark
2686          * pinned objects as we go, memzero() the empty fragments so they are ready for the
2687          * next allocations.
2688          */
2689         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2690         fragment_total = sgen_build_nursery_fragments (nursery_section,
2691                         nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries,
2692                         unpin_queue);
2693         if (!fragment_total)
2694                 degraded_mode = 1;
2695
2696         /* Clear TLABs for all threads */
2697         sgen_clear_tlabs ();
2698
2699         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2700         TV_GETTIME (btv);
2701         time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2702         SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2703
2704         if (consistency_check_at_minor_collection)
2705                 sgen_check_major_refs ();
2706
2707         major_collector.finish_nursery_collection ();
2708
2709         TV_GETTIME (all_btv);
2710         gc_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2711
2712         if (heap_dump_file)
2713                 dump_heap ("minor", stat_minor_gcs - 1, NULL);
2714
2715         /* prepare the pin queue for the next collection */
2716         sgen_finish_pinning ();
2717         if (fin_ready_list || critical_fin_list) {
2718                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2719                 mono_gc_finalize_notify ();
2720         }
2721         sgen_pin_stats_reset ();
2722         /* clear cemented hash */
2723         sgen_cement_clear_below_threshold ();
2724
2725         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2726
2727         remset.finish_minor_collection ();
2728
2729         check_scan_starts ();
2730
2731         binary_protocol_flush_buffers (FALSE);
2732
2733         sgen_memgov_minor_collection_end ();
2734
2735         /*objects are late pinned because of lack of memory, so a major is a good call*/
2736         needs_major = objects_pinned > 0;
2737         current_collection_generation = -1;
2738         objects_pinned = 0;
2739
2740         MONO_GC_END (GENERATION_NURSERY);
2741         binary_protocol_collection_end (stat_minor_gcs - 1, GENERATION_NURSERY);
2742
2743         if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2744                 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2745
2746         return needs_major;
2747 }
2748
2749 static void
2750 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2751 {
2752         ctx->scan_func (obj, ctx->queue);
2753 }
2754
2755 static void
2756 scan_nursery_objects (ScanCopyContext ctx)
2757 {
2758         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2759                         (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2760 }
2761
2762 static void
2763 major_copy_or_mark_from_roots (int *old_next_pin_slot, gboolean finish_up_concurrent_mark, gboolean scan_mod_union)
2764 {
2765         LOSObject *bigobj;
2766         TV_DECLARE (atv);
2767         TV_DECLARE (btv);
2768         /* FIXME: only use these values for the precise scan
2769          * note that to_space pointers should be excluded anyway...
2770          */
2771         char *heap_start = NULL;
2772         char *heap_end = (char*)-1;
2773         gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2774         GCRootReport root_report = { 0 };
2775         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2776         ScanThreadDataJobData *stdjd;
2777         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2778         ScanCopyContext ctx;
2779
2780         if (concurrent_collection_in_progress) {
2781                 /*This cleans up unused fragments */
2782                 sgen_nursery_allocator_prepare_for_pinning ();
2783
2784                 if (do_concurrent_checks)
2785                         check_nursery_is_clean ();
2786         } else {
2787                 /* The concurrent collector doesn't touch the nursery. */
2788                 sgen_nursery_alloc_prepare_for_major ();
2789         }
2790
2791         init_gray_queue ();
2792
2793         TV_GETTIME (atv);
2794
2795         /* Pinning depends on this */
2796         sgen_clear_nursery_fragments ();
2797
2798         if (whole_heap_check_before_collection)
2799                 sgen_check_whole_heap (finish_up_concurrent_mark);
2800
2801         TV_GETTIME (btv);
2802         time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2803
2804         if (!sgen_collection_is_concurrent ())
2805                 nursery_section->next_data = sgen_get_nursery_end ();
2806         /* we should also coalesce scanning from sections close to each other
2807          * and deal with pointers outside of the sections later.
2808          */
2809
2810         objects_pinned = 0;
2811         *major_collector.have_swept = FALSE;
2812
2813         if (xdomain_checks) {
2814                 sgen_clear_nursery_fragments ();
2815                 check_for_xdomain_refs ();
2816         }
2817
2818         if (!concurrent_collection_in_progress) {
2819                 /* Remsets are not useful for a major collection */
2820                 remset.prepare_for_major_collection ();
2821         }
2822
2823         sgen_process_fin_stage_entries ();
2824         sgen_process_dislink_stage_entries ();
2825
2826         TV_GETTIME (atv);
2827         sgen_init_pinning ();
2828         SGEN_LOG (6, "Collecting pinned addresses");
2829         pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2830
2831         if (!concurrent_collection_in_progress || finish_up_concurrent_mark) {
2832                 if (major_collector.is_concurrent) {
2833                         /*
2834                          * The concurrent major collector cannot evict
2835                          * yet, so we need to pin cemented objects to
2836                          * not break some asserts.
2837                          *
2838                          * FIXME: We could evict now!
2839                          */
2840                         sgen_cement_iterate (pin_stage_object_callback, NULL);
2841                 }
2842
2843                 if (!concurrent_collection_in_progress)
2844                         sgen_cement_reset ();
2845         }
2846
2847         sgen_optimize_pin_queue (0);
2848
2849         /*
2850          * The concurrent collector doesn't move objects, neither on
2851          * the major heap nor in the nursery, so we can mark even
2852          * before pinning has finished.  For the non-concurrent
2853          * collector we start the workers after pinning.
2854          */
2855         if (concurrent_collection_in_progress) {
2856                 sgen_workers_start_all_workers ();
2857                 sgen_workers_start_marking ();
2858         }
2859
2860         /*
2861          * pin_queue now contains all candidate pointers, sorted and
2862          * uniqued.  We must do two passes now to figure out which
2863          * objects are pinned.
2864          *
2865          * The first is to find within the pin_queue the area for each
2866          * section.  This requires that the pin_queue be sorted.  We
2867          * also process the LOS objects and pinned chunks here.
2868          *
2869          * The second, destructive, pass is to reduce the section
2870          * areas to pointers to the actually pinned objects.
2871          */
2872         SGEN_LOG (6, "Pinning from sections");
2873         /* first pass for the sections */
2874         sgen_find_section_pin_queue_start_end (nursery_section);
2875         major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2876         /* identify possible pointers to the insize of large objects */
2877         SGEN_LOG (6, "Pinning from large objects");
2878         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2879                 int dummy;
2880                 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) {
2881                         binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2882
2883 #ifdef ENABLE_DTRACE
2884                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2885                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2886                                 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2887                         }
2888 #endif
2889
2890                         if (sgen_los_object_is_pinned (bigobj->data)) {
2891                                 g_assert (finish_up_concurrent_mark);
2892                                 continue;
2893                         }
2894                         sgen_los_pin_object (bigobj->data);
2895                         /* FIXME: only enqueue if object has references */
2896                         GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data);
2897                         if (G_UNLIKELY (do_pin_stats))
2898                                 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2899                         SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2900
2901                         if (profile_roots)
2902                                 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2903                 }
2904         }
2905         if (profile_roots)
2906                 notify_gc_roots (&root_report);
2907         /* second pass for the sections */
2908         ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2909         ctx.copy_func = NULL;
2910         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2911
2912         /*
2913          * Concurrent mark never follows references into the nursery.
2914          * In the start and finish pauses we must scan live nursery
2915          * objects, though.  We could simply scan all nursery objects,
2916          * but that would be conservative.  The easiest way is to do a
2917          * nursery collection, which copies all live nursery objects
2918          * (except pinned ones, with the simple nursery) to the major
2919          * heap.  Scanning the mod union table later will then scan
2920          * those promoted objects, provided they're reachable.  Pinned
2921          * objects in the nursery - which we can trivially find in the
2922          * pinning queue - are treated as roots in the mark pauses.
2923          *
2924          * The split nursery complicates the latter part because
2925          * non-pinned objects can survive in the nursery.  That's why
2926          * we need to do a full front-to-back scan of the nursery,
2927          * marking all objects.
2928          *
2929          * Non-concurrent mark evacuates from the nursery, so it's
2930          * sufficient to just scan pinned nursery objects.
2931          */
2932         if (concurrent_collection_in_progress && sgen_minor_collector.is_split) {
2933                 scan_nursery_objects (ctx);
2934         } else {
2935                 sgen_pin_objects_in_section (nursery_section, ctx);
2936                 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2937                         sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2938         }
2939
2940         major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2941         if (old_next_pin_slot)
2942                 *old_next_pin_slot = sgen_get_pinned_count ();
2943
2944         TV_GETTIME (btv);
2945         time_major_pinning += TV_ELAPSED (atv, btv);
2946         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2947         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2948
2949         major_collector.init_to_space ();
2950
2951 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2952         main_gc_thread = mono_native_thread_self ();
2953 #endif
2954
2955         if (!concurrent_collection_in_progress && major_collector.is_parallel) {
2956                 sgen_workers_start_all_workers ();
2957                 sgen_workers_start_marking ();
2958         }
2959
2960         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2961                 report_registered_roots ();
2962         TV_GETTIME (atv);
2963         time_major_scan_pinned += TV_ELAPSED (btv, atv);
2964
2965         /* registered roots, this includes static fields */
2966         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2967         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2968         scrrjd_normal->scan_func = current_object_ops.scan_object;
2969         scrrjd_normal->heap_start = heap_start;
2970         scrrjd_normal->heap_end = heap_end;
2971         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2972         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2973
2974         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2975         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2976         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2977         scrrjd_wbarrier->heap_start = heap_start;
2978         scrrjd_wbarrier->heap_end = heap_end;
2979         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2980         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2981
2982         TV_GETTIME (btv);
2983         time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
2984
2985         /* Threads */
2986         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2987         stdjd->heap_start = heap_start;
2988         stdjd->heap_end = heap_end;
2989         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2990
2991         TV_GETTIME (atv);
2992         time_major_scan_thread_data += TV_ELAPSED (btv, atv);
2993
2994         TV_GETTIME (btv);
2995         time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
2996
2997         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2998                 report_finalizer_roots ();
2999
3000         /* scan the list of objects ready for finalization */
3001         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3002         sfejd_fin_ready->list = fin_ready_list;
3003         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
3004
3005         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3006         sfejd_critical_fin->list = critical_fin_list;
3007         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
3008
3009         if (scan_mod_union) {
3010                 g_assert (finish_up_concurrent_mark);
3011
3012                 /* Mod union card table */
3013                 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
3014                 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
3015         }
3016
3017         TV_GETTIME (atv);
3018         time_major_scan_finalized += TV_ELAPSED (btv, atv);
3019         SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
3020
3021         TV_GETTIME (btv);
3022         time_major_scan_big_objects += TV_ELAPSED (atv, btv);
3023
3024         if (concurrent_collection_in_progress) {
3025                 /* prepare the pin queue for the next collection */
3026                 sgen_finish_pinning ();
3027
3028                 sgen_pin_stats_reset ();
3029
3030                 if (do_concurrent_checks)
3031                         check_nursery_is_clean ();
3032         }
3033 }
3034
3035 static void
3036 major_start_collection (gboolean concurrent, int *old_next_pin_slot)
3037 {
3038         MONO_GC_BEGIN (GENERATION_OLD);
3039         binary_protocol_collection_begin (stat_major_gcs, GENERATION_OLD);
3040
3041         current_collection_generation = GENERATION_OLD;
3042 #ifndef DISABLE_PERFCOUNTERS
3043         mono_perfcounters->gc_collections1++;
3044 #endif
3045
3046         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3047
3048         if (concurrent) {
3049                 g_assert (major_collector.is_concurrent);
3050                 concurrent_collection_in_progress = TRUE;
3051
3052                 sgen_cement_concurrent_start ();
3053
3054                 current_object_ops = major_collector.major_concurrent_ops;
3055         } else {
3056                 current_object_ops = major_collector.major_ops;
3057         }
3058
3059         reset_pinned_from_failed_allocation ();
3060
3061         sgen_memgov_major_collection_start ();
3062
3063         //count_ref_nonref_objs ();
3064         //consistency_check ();
3065
3066         check_scan_starts ();
3067
3068         degraded_mode = 0;
3069         SGEN_LOG (1, "Start major collection %d", stat_major_gcs);
3070         stat_major_gcs++;
3071         gc_stats.major_gc_count ++;
3072
3073         if (major_collector.start_major_collection)
3074                 major_collector.start_major_collection ();
3075
3076         major_copy_or_mark_from_roots (old_next_pin_slot, FALSE, FALSE);
3077 }
3078
3079 static void
3080 wait_for_workers_to_finish (void)
3081 {
3082         if (concurrent_collection_in_progress || major_collector.is_parallel) {
3083                 gray_queue_redirect (&gray_queue);
3084                 sgen_workers_join ();
3085         }
3086
3087         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3088
3089 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
3090         main_gc_thread = NULL;
3091 #endif
3092 }
3093
3094 static void
3095 major_finish_collection (const char *reason, int old_next_pin_slot, gboolean scan_mod_union)
3096 {
3097         LOSObject *bigobj, *prevbo;
3098         TV_DECLARE (atv);
3099         TV_DECLARE (btv);
3100
3101         TV_GETTIME (btv);
3102
3103         if (concurrent_collection_in_progress || major_collector.is_parallel)
3104                 wait_for_workers_to_finish ();
3105
3106         if (concurrent_collection_in_progress) {
3107                 current_object_ops = major_collector.major_concurrent_ops;
3108
3109                 major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
3110                 wait_for_workers_to_finish ();
3111
3112                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3113
3114                 if (do_concurrent_checks)
3115                         check_nursery_is_clean ();
3116         } else {
3117                 current_object_ops = major_collector.major_ops;
3118         }
3119
3120         /*
3121          * The workers have stopped so we need to finish gray queue
3122          * work that might result from finalization in the main GC
3123          * thread.  Redirection must therefore be turned off.
3124          */
3125         sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
3126         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3127
3128         /* all the objects in the heap */
3129         finish_gray_stack (GENERATION_OLD, &gray_queue);
3130         TV_GETTIME (atv);
3131         time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
3132
3133         /*
3134          * The (single-threaded) finalization code might have done
3135          * some copying/marking so we can only reset the GC thread's
3136          * worker data here instead of earlier when we joined the
3137          * workers.
3138          */
3139         sgen_workers_reset_data ();
3140
3141         if (objects_pinned) {
3142                 g_assert (!concurrent_collection_in_progress);
3143
3144                 /*This is slow, but we just OOM'd*/
3145                 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
3146                 sgen_optimize_pin_queue (0);
3147                 sgen_find_section_pin_queue_start_end (nursery_section);
3148                 objects_pinned = 0;
3149         }
3150
3151         reset_heap_boundaries ();
3152         sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
3153
3154         if (check_mark_bits_after_major_collection)
3155                 sgen_check_major_heap_marked ();
3156
3157         MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
3158
3159         /* sweep the big objects list */
3160         prevbo = NULL;
3161         for (bigobj = los_object_list; bigobj;) {
3162                 g_assert (!object_is_pinned (bigobj->data));
3163                 if (sgen_los_object_is_pinned (bigobj->data)) {
3164                         sgen_los_unpin_object (bigobj->data);
3165                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
3166                 } else {
3167                         LOSObject *to_free;
3168                         /* not referenced anywhere, so we can free it */
3169                         if (prevbo)
3170                                 prevbo->next = bigobj->next;
3171                         else
3172                                 los_object_list = bigobj->next;
3173                         to_free = bigobj;
3174                         bigobj = bigobj->next;
3175                         sgen_los_free_object (to_free);
3176                         continue;
3177                 }
3178                 prevbo = bigobj;
3179                 bigobj = bigobj->next;
3180         }
3181
3182         TV_GETTIME (btv);
3183         time_major_free_bigobjs += TV_ELAPSED (atv, btv);
3184
3185         sgen_los_sweep ();
3186
3187         TV_GETTIME (atv);
3188         time_major_los_sweep += TV_ELAPSED (btv, atv);
3189
3190         major_collector.sweep ();
3191
3192         MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
3193
3194         TV_GETTIME (btv);
3195         time_major_sweep += TV_ELAPSED (atv, btv);
3196
3197         if (!concurrent_collection_in_progress) {
3198                 /* walk the pin_queue, build up the fragment list of free memory, unmark
3199                  * pinned objects as we go, memzero() the empty fragments so they are ready for the
3200                  * next allocations.
3201                  */
3202                 if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries, NULL))
3203                         degraded_mode = 1;
3204
3205                 /* prepare the pin queue for the next collection */
3206                 sgen_finish_pinning ();
3207
3208                 /* Clear TLABs for all threads */
3209                 sgen_clear_tlabs ();
3210
3211                 sgen_pin_stats_reset ();
3212         }
3213
3214         if (concurrent_collection_in_progress)
3215                 sgen_cement_concurrent_finish ();
3216         sgen_cement_clear_below_threshold ();
3217
3218         TV_GETTIME (atv);
3219         time_major_fragment_creation += TV_ELAPSED (btv, atv);
3220
3221         if (heap_dump_file)
3222                 dump_heap ("major", stat_major_gcs - 1, reason);
3223
3224         if (fin_ready_list || critical_fin_list) {
3225                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
3226                 mono_gc_finalize_notify ();
3227         }
3228
3229         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3230
3231         sgen_memgov_major_collection_end ();
3232         current_collection_generation = -1;
3233
3234         major_collector.finish_major_collection ();
3235
3236         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3237
3238         if (concurrent_collection_in_progress)
3239                 concurrent_collection_in_progress = FALSE;
3240
3241         check_scan_starts ();
3242
3243         binary_protocol_flush_buffers (FALSE);
3244
3245         //consistency_check ();
3246
3247         MONO_GC_END (GENERATION_OLD);
3248         binary_protocol_collection_end (stat_major_gcs - 1, GENERATION_OLD);
3249 }
3250
3251 static gboolean
3252 major_do_collection (const char *reason)
3253 {
3254         TV_DECLARE (all_atv);
3255         TV_DECLARE (all_btv);
3256         int old_next_pin_slot;
3257
3258         if (major_collector.get_and_reset_num_major_objects_marked) {
3259                 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
3260                 g_assert (!num_marked);
3261         }
3262
3263         /* world must be stopped already */
3264         TV_GETTIME (all_atv);
3265
3266         major_start_collection (FALSE, &old_next_pin_slot);
3267         major_finish_collection (reason, old_next_pin_slot, FALSE);
3268
3269         TV_GETTIME (all_btv);
3270         gc_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3271
3272         /* FIXME: also report this to the user, preferably in gc-end. */
3273         if (major_collector.get_and_reset_num_major_objects_marked)
3274                 major_collector.get_and_reset_num_major_objects_marked ();
3275
3276         return bytes_pinned_from_failed_allocation > 0;
3277 }
3278
3279 static gboolean major_do_collection (const char *reason);
3280
3281 static void
3282 major_start_concurrent_collection (const char *reason)
3283 {
3284         long long num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3285
3286         g_assert (num_objects_marked == 0);
3287
3288         MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
3289
3290         // FIXME: store reason and pass it when finishing
3291         major_start_collection (TRUE, NULL);
3292
3293         gray_queue_redirect (&gray_queue);
3294         sgen_workers_wait_for_jobs ();
3295
3296         num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3297         MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
3298
3299         current_collection_generation = -1;
3300 }
3301
3302 static gboolean
3303 major_update_or_finish_concurrent_collection (gboolean force_finish)
3304 {
3305         SgenGrayQueue unpin_queue;
3306         memset (&unpin_queue, 0, sizeof (unpin_queue));
3307
3308         MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3309
3310         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3311
3312         major_collector.update_cardtable_mod_union ();
3313         sgen_los_update_cardtable_mod_union ();
3314
3315         if (!force_finish && !sgen_workers_all_done ()) {
3316                 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3317                 return FALSE;
3318         }
3319
3320         if (mod_union_consistency_check)
3321                 sgen_check_mod_union_consistency ();
3322
3323         collect_nursery (&unpin_queue, TRUE);
3324
3325         current_collection_generation = GENERATION_OLD;
3326         major_finish_collection ("finishing", -1, TRUE);
3327
3328         if (whole_heap_check_before_collection)
3329                 sgen_check_whole_heap (FALSE);
3330
3331         unpin_objects_from_queue (&unpin_queue);
3332         sgen_gray_object_queue_deinit (&unpin_queue);
3333
3334         MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3335
3336         current_collection_generation = -1;
3337
3338         return TRUE;
3339 }
3340
3341 /*
3342  * Ensure an allocation request for @size will succeed by freeing enough memory.
3343  *
3344  * LOCKING: The GC lock MUST be held.
3345  */
3346 void
3347 sgen_ensure_free_space (size_t size)
3348 {
3349         int generation_to_collect = -1;
3350         const char *reason = NULL;
3351
3352
3353         if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3354                 if (sgen_need_major_collection (size)) {
3355                         reason = "LOS overflow";
3356                         generation_to_collect = GENERATION_OLD;
3357                 }
3358         } else {
3359                 if (degraded_mode) {
3360                         if (sgen_need_major_collection (size)) {
3361                                 reason = "Degraded mode overflow";
3362                                 generation_to_collect = GENERATION_OLD;
3363                         }
3364                 } else if (sgen_need_major_collection (size)) {
3365                         reason = "Minor allowance";
3366                         generation_to_collect = GENERATION_OLD;
3367                 } else {
3368                         generation_to_collect = GENERATION_NURSERY;
3369                         reason = "Nursery full";
3370                 }
3371         }
3372
3373         if (generation_to_collect == -1) {
3374                 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3375                         generation_to_collect = GENERATION_OLD;
3376                         reason = "Finish concurrent collection";
3377                 }
3378         }
3379
3380         if (generation_to_collect == -1)
3381                 return;
3382         sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3383 }
3384
3385 void
3386 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3387 {
3388         TV_DECLARE (gc_end);
3389         GGTimingInfo infos [2];
3390         int overflow_generation_to_collect = -1;
3391         int oldest_generation_collected = generation_to_collect;
3392         const char *overflow_reason = NULL;
3393
3394         MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3395         if (wait_to_finish)
3396                 binary_protocol_collection_force (generation_to_collect);
3397
3398         g_assert (generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD);
3399
3400         memset (infos, 0, sizeof (infos));
3401         mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3402
3403         infos [0].generation = generation_to_collect;
3404         infos [0].reason = reason;
3405         infos [0].is_overflow = FALSE;
3406         TV_GETTIME (infos [0].total_time);
3407         infos [1].generation = -1;
3408
3409         sgen_stop_world (generation_to_collect);
3410
3411         if (concurrent_collection_in_progress) {
3412                 if (major_update_or_finish_concurrent_collection (wait_to_finish && generation_to_collect == GENERATION_OLD)) {
3413                         oldest_generation_collected = GENERATION_OLD;
3414                         goto done;
3415                 }
3416                 if (generation_to_collect == GENERATION_OLD)
3417                         goto done;
3418         } else {
3419                 if (generation_to_collect == GENERATION_OLD &&
3420                                 allow_synchronous_major &&
3421                                 major_collector.want_synchronous_collection &&
3422                                 *major_collector.want_synchronous_collection) {
3423                         wait_to_finish = TRUE;
3424                 }
3425         }
3426
3427         //FIXME extract overflow reason
3428         if (generation_to_collect == GENERATION_NURSERY) {
3429                 if (collect_nursery (NULL, FALSE)) {
3430                         overflow_generation_to_collect = GENERATION_OLD;
3431                         overflow_reason = "Minor overflow";
3432                 }
3433         } else {
3434                 if (major_collector.is_concurrent) {
3435                         g_assert (!concurrent_collection_in_progress);
3436                         if (!wait_to_finish)
3437                                 collect_nursery (NULL, FALSE);
3438                 }
3439
3440                 if (major_collector.is_concurrent && !wait_to_finish) {
3441                         major_start_concurrent_collection (reason);
3442                         // FIXME: set infos[0] properly
3443                         goto done;
3444                 } else {
3445                         if (major_do_collection (reason)) {
3446                                 overflow_generation_to_collect = GENERATION_NURSERY;
3447                                 overflow_reason = "Excessive pinning";
3448                         }
3449                 }
3450         }
3451
3452         TV_GETTIME (gc_end);
3453         infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end);
3454
3455
3456         if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) {
3457                 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3458                 infos [1].generation = overflow_generation_to_collect;
3459                 infos [1].reason = overflow_reason;
3460                 infos [1].is_overflow = TRUE;
3461                 infos [1].total_time = gc_end;
3462
3463                 if (overflow_generation_to_collect == GENERATION_NURSERY)
3464                         collect_nursery (NULL, FALSE);
3465                 else
3466                         major_do_collection (overflow_reason);
3467
3468                 TV_GETTIME (gc_end);
3469                 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3470
3471                 /* keep events symmetric */
3472                 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3473
3474                 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3475         }
3476
3477         SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3478
3479         /* this also sets the proper pointers for the next allocation */
3480         if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3481                 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3482                 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%d pinned)", requested_size, sgen_get_pinned_count ());
3483                 sgen_dump_pin_queue ();
3484                 degraded_mode = 1;
3485         }
3486
3487  done:
3488         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3489
3490         sgen_restart_world (oldest_generation_collected, infos);
3491
3492         mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3493 }
3494
3495 /*
3496  * ######################################################################
3497  * ########  Memory allocation from the OS
3498  * ######################################################################
3499  * This section of code deals with getting memory from the OS and
3500  * allocating memory for GC-internal data structures.
3501  * Internal memory can be handled with a freelist for small objects.
3502  */
3503
3504 /*
3505  * Debug reporting.
3506  */
3507 G_GNUC_UNUSED static void
3508 report_internal_mem_usage (void)
3509 {
3510         printf ("Internal memory usage:\n");
3511         sgen_report_internal_mem_usage ();
3512         printf ("Pinned memory usage:\n");
3513         major_collector.report_pinned_memory_usage ();
3514 }
3515
3516 /*
3517  * ######################################################################
3518  * ########  Finalization support
3519  * ######################################################################
3520  */
3521
3522 static inline gboolean
3523 sgen_major_is_object_alive (void *object)
3524 {
3525         mword objsize;
3526
3527         /* Oldgen objects can be pinned and forwarded too */
3528         if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3529                 return TRUE;
3530
3531         /*
3532          * FIXME: major_collector.is_object_live() also calculates the
3533          * size.  Avoid the double calculation.
3534          */
3535         objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3536         if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3537                 return sgen_los_object_is_pinned (object);
3538
3539         return major_collector.is_object_live (object);
3540 }
3541
3542 /*
3543  * If the object has been forwarded it means it's still referenced from a root.
3544  * If it is pinned it's still alive as well.
3545  * A LOS object is only alive if we have pinned it.
3546  * Return TRUE if @obj is ready to be finalized.
3547  */
3548 static inline gboolean
3549 sgen_is_object_alive (void *object)
3550 {
3551         if (ptr_in_nursery (object))
3552                 return sgen_nursery_is_object_alive (object);
3553
3554         return sgen_major_is_object_alive (object);
3555 }
3556
3557 /*
3558  * This function returns true if @object is either alive or it belongs to the old gen
3559  * and we're currently doing a minor collection.
3560  */
3561 static inline int
3562 sgen_is_object_alive_for_current_gen (char *object)
3563 {
3564         if (ptr_in_nursery (object))
3565                 return sgen_nursery_is_object_alive (object);
3566
3567         if (current_collection_generation == GENERATION_NURSERY)
3568                 return TRUE;
3569
3570         return sgen_major_is_object_alive (object);
3571 }
3572
3573 /*
3574  * This function returns true if @object is either alive and belongs to the
3575  * current collection - major collections are full heap, so old gen objects
3576  * are never alive during a minor collection.
3577  */
3578 static inline int
3579 sgen_is_object_alive_and_on_current_collection (char *object)
3580 {
3581         if (ptr_in_nursery (object))
3582                 return sgen_nursery_is_object_alive (object);
3583
3584         if (current_collection_generation == GENERATION_NURSERY)
3585                 return FALSE;
3586
3587         return sgen_major_is_object_alive (object);
3588 }
3589
3590
3591 gboolean
3592 sgen_gc_is_object_ready_for_finalization (void *object)
3593 {
3594         return !sgen_is_object_alive (object);
3595 }
3596
3597 static gboolean
3598 has_critical_finalizer (MonoObject *obj)
3599 {
3600         MonoClass *class;
3601
3602         if (!mono_defaults.critical_finalizer_object)
3603                 return FALSE;
3604
3605         class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3606
3607         return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3608 }
3609
3610 void
3611 sgen_queue_finalization_entry (MonoObject *obj)
3612 {
3613         FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3614         gboolean critical = has_critical_finalizer (obj);
3615         entry->object = obj;
3616         if (critical) {
3617                 entry->next = critical_fin_list;
3618                 critical_fin_list = entry;
3619         } else {
3620                 entry->next = fin_ready_list;
3621                 fin_ready_list = entry;
3622         }
3623
3624 #ifdef ENABLE_DTRACE
3625         if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3626                 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3627                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3628                 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3629                                 vt->klass->name_space, vt->klass->name, gen, critical);
3630         }
3631 #endif
3632 }
3633
3634 gboolean
3635 sgen_object_is_live (void *obj)
3636 {
3637         return sgen_is_object_alive_and_on_current_collection (obj);
3638 }
3639
3640 /* LOCKING: requires that the GC lock is held */
3641 static void
3642 null_ephemerons_for_domain (MonoDomain *domain)
3643 {
3644         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3645
3646         while (current) {
3647                 MonoObject *object = (MonoObject*)current->array;
3648
3649                 if (object && !object->vtable) {
3650                         EphemeronLinkNode *tmp = current;
3651
3652                         if (prev)
3653                                 prev->next = current->next;
3654                         else
3655                                 ephemeron_list = current->next;
3656
3657                         current = current->next;
3658                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3659                 } else {
3660                         prev = current;
3661                         current = current->next;
3662                 }
3663         }
3664 }
3665
3666 /* LOCKING: requires that the GC lock is held */
3667 static void
3668 clear_unreachable_ephemerons (ScanCopyContext ctx)
3669 {
3670         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3671         GrayQueue *queue = ctx.queue;
3672         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3673         MonoArray *array;
3674         Ephemeron *cur, *array_end;
3675         char *tombstone;
3676
3677         while (current) {
3678                 char *object = current->array;
3679
3680                 if (!sgen_is_object_alive_for_current_gen (object)) {
3681                         EphemeronLinkNode *tmp = current;
3682
3683                         SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3684
3685                         if (prev)
3686                                 prev->next = current->next;
3687                         else
3688                                 ephemeron_list = current->next;
3689
3690                         current = current->next;
3691                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3692
3693                         continue;
3694                 }
3695
3696                 copy_func ((void**)&object, queue);
3697                 current->array = object;
3698
3699                 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3700
3701                 array = (MonoArray*)object;
3702                 cur = mono_array_addr (array, Ephemeron, 0);
3703                 array_end = cur + mono_array_length_fast (array);
3704                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3705
3706                 for (; cur < array_end; ++cur) {
3707                         char *key = (char*)cur->key;
3708
3709                         if (!key || key == tombstone)
3710                                 continue;
3711
3712                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3713                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3714                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3715
3716                         if (!sgen_is_object_alive_for_current_gen (key)) {
3717                                 cur->key = tombstone;
3718                                 cur->value = NULL;
3719                                 continue;
3720                         }
3721                 }
3722                 prev = current;
3723                 current = current->next;
3724         }
3725 }
3726
3727 /*
3728 LOCKING: requires that the GC lock is held
3729
3730 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3731 */
3732 static int
3733 mark_ephemerons_in_range (ScanCopyContext ctx)
3734 {
3735         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3736         GrayQueue *queue = ctx.queue;
3737         int nothing_marked = 1;
3738         EphemeronLinkNode *current = ephemeron_list;
3739         MonoArray *array;
3740         Ephemeron *cur, *array_end;
3741         char *tombstone;
3742
3743         for (current = ephemeron_list; current; current = current->next) {
3744                 char *object = current->array;
3745                 SGEN_LOG (5, "Ephemeron array at %p", object);
3746
3747                 /*It has to be alive*/
3748                 if (!sgen_is_object_alive_for_current_gen (object)) {
3749                         SGEN_LOG (5, "\tnot reachable");
3750                         continue;
3751                 }
3752
3753                 copy_func ((void**)&object, queue);
3754
3755                 array = (MonoArray*)object;
3756                 cur = mono_array_addr (array, Ephemeron, 0);
3757                 array_end = cur + mono_array_length_fast (array);
3758                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3759
3760                 for (; cur < array_end; ++cur) {
3761                         char *key = cur->key;
3762
3763                         if (!key || key == tombstone)
3764                                 continue;
3765
3766                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3767                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3768                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3769
3770                         if (sgen_is_object_alive_for_current_gen (key)) {
3771                                 char *value = cur->value;
3772
3773                                 copy_func ((void**)&cur->key, queue);
3774                                 if (value) {
3775                                         if (!sgen_is_object_alive_for_current_gen (value))
3776                                                 nothing_marked = 0;
3777                                         copy_func ((void**)&cur->value, queue);
3778                                 }
3779                         }
3780                 }
3781         }
3782
3783         SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3784         return nothing_marked;
3785 }
3786
3787 int
3788 mono_gc_invoke_finalizers (void)
3789 {
3790         FinalizeReadyEntry *entry = NULL;
3791         gboolean entry_is_critical = FALSE;
3792         int count = 0;
3793         void *obj;
3794         /* FIXME: batch to reduce lock contention */
3795         while (fin_ready_list || critical_fin_list) {
3796                 LOCK_GC;
3797
3798                 if (entry) {
3799                         FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3800
3801                         /* We have finalized entry in the last
3802                            interation, now we need to remove it from
3803                            the list. */
3804                         if (*list == entry)
3805                                 *list = entry->next;
3806                         else {
3807                                 FinalizeReadyEntry *e = *list;
3808                                 while (e->next != entry)
3809                                         e = e->next;
3810                                 e->next = entry->next;
3811                         }
3812                         sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3813                         entry = NULL;
3814                 }
3815
3816                 /* Now look for the first non-null entry. */
3817                 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3818                         ;
3819                 if (entry) {
3820                         entry_is_critical = FALSE;
3821                 } else {
3822                         entry_is_critical = TRUE;
3823                         for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3824                                 ;
3825                 }
3826
3827                 if (entry) {
3828                         g_assert (entry->object);
3829                         num_ready_finalizers--;
3830                         obj = entry->object;
3831                         entry->object = NULL;
3832                         SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3833                 }
3834
3835                 UNLOCK_GC;
3836
3837                 if (!entry)
3838                         break;
3839
3840                 g_assert (entry->object == NULL);
3841                 count++;
3842                 /* the object is on the stack so it is pinned */
3843                 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3844                 mono_gc_run_finalize (obj, NULL);
3845         }
3846         g_assert (!entry);
3847         return count;
3848 }
3849
3850 gboolean
3851 mono_gc_pending_finalizers (void)
3852 {
3853         return fin_ready_list || critical_fin_list;
3854 }
3855
3856 /*
3857  * ######################################################################
3858  * ########  registered roots support
3859  * ######################################################################
3860  */
3861
3862 /*
3863  * We do not coalesce roots.
3864  */
3865 static int
3866 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3867 {
3868         RootRecord new_root;
3869         int i;
3870         LOCK_GC;
3871         for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3872                 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3873                 /* we allow changing the size and the descriptor (for thread statics etc) */
3874                 if (root) {
3875                         size_t old_size = root->end_root - start;
3876                         root->end_root = start + size;
3877                         g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3878                                           ((root->root_desc == 0) && (descr == NULL)));
3879                         root->root_desc = (mword)descr;
3880                         roots_size += size;
3881                         roots_size -= old_size;
3882                         UNLOCK_GC;
3883                         return TRUE;
3884                 }
3885         }
3886
3887         new_root.end_root = start + size;
3888         new_root.root_desc = (mword)descr;
3889
3890         sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3891         roots_size += size;
3892
3893         SGEN_LOG (3, "Added root for range: %p-%p, descr: %p  (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3894
3895         UNLOCK_GC;
3896         return TRUE;
3897 }
3898
3899 int
3900 mono_gc_register_root (char *start, size_t size, void *descr)
3901 {
3902         return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3903 }
3904
3905 int
3906 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3907 {
3908         return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3909 }
3910
3911 void
3912 mono_gc_deregister_root (char* addr)
3913 {
3914         int root_type;
3915         RootRecord root;
3916
3917         LOCK_GC;
3918         for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3919                 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3920                         roots_size -= (root.end_root - addr);
3921         }
3922         UNLOCK_GC;
3923 }
3924
3925 /*
3926  * ######################################################################
3927  * ########  Thread handling (stop/start code)
3928  * ######################################################################
3929  */
3930
3931 unsigned int sgen_global_stop_count = 0;
3932
3933 int
3934 sgen_get_current_collection_generation (void)
3935 {
3936         return current_collection_generation;
3937 }
3938
3939 void
3940 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3941 {
3942         gc_callbacks = *callbacks;
3943 }
3944
3945 MonoGCCallbacks *
3946 mono_gc_get_gc_callbacks ()
3947 {
3948         return &gc_callbacks;
3949 }
3950
3951 /* Variables holding start/end nursery so it won't have to be passed at every call */
3952 static void *scan_area_arg_start, *scan_area_arg_end;
3953
3954 void
3955 mono_gc_conservatively_scan_area (void *start, void *end)
3956 {
3957         conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
3958 }
3959
3960 void*
3961 mono_gc_scan_object (void *obj)
3962 {
3963         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
3964         current_object_ops.copy_or_mark_object (&obj, data->queue);
3965         return obj;
3966 }
3967
3968 /*
3969  * Mark from thread stacks and registers.
3970  */
3971 static void
3972 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
3973 {
3974         SgenThreadInfo *info;
3975
3976         scan_area_arg_start = start_nursery;
3977         scan_area_arg_end = end_nursery;
3978
3979         FOREACH_THREAD (info) {
3980                 if (info->skip) {
3981                         SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3982                         continue;
3983                 }
3984                 if (info->gc_disabled) {
3985                         SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3986                         continue;
3987                 }
3988
3989                 if (!info->joined_stw) {
3990                         SGEN_LOG (3, "Skipping thread not seen in STW %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3991                         continue;
3992                 }
3993
3994                 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
3995                 if (!info->thread_is_dying) {
3996                         if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
3997                                 UserCopyOrMarkData data = { NULL, queue };
3998                                 set_user_copy_or_mark_data (&data);
3999                                 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
4000                                 set_user_copy_or_mark_data (NULL);
4001                         } else if (!precise) {
4002                                 if (!conservative_stack_mark) {
4003                                         fprintf (stderr, "Precise stack mark not supported - disabling.\n");
4004                                         conservative_stack_mark = TRUE;
4005                                 }
4006                                 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
4007                         }
4008                 }
4009
4010                 if (!info->thread_is_dying && !precise) {
4011 #ifdef USE_MONO_CTX
4012                         conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
4013                                 start_nursery, end_nursery, PIN_TYPE_STACK);
4014 #else
4015                         conservatively_pin_objects_from (&info->regs, &info->regs + ARCH_NUM_REGS,
4016                                         start_nursery, end_nursery, PIN_TYPE_STACK);
4017 #endif
4018                 }
4019         } END_FOREACH_THREAD
4020 }
4021
4022 static gboolean
4023 ptr_on_stack (void *ptr)
4024 {
4025         gpointer stack_start = &stack_start;
4026         SgenThreadInfo *info = mono_thread_info_current ();
4027
4028         if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
4029                 return TRUE;
4030         return FALSE;
4031 }
4032
4033 static void*
4034 sgen_thread_register (SgenThreadInfo* info, void *addr)
4035 {
4036         LOCK_GC;
4037 #ifndef HAVE_KW_THREAD
4038         info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
4039
4040         g_assert (!mono_native_tls_get_value (thread_info_key));
4041         mono_native_tls_set_value (thread_info_key, info);
4042 #else
4043         sgen_thread_info = info;
4044 #endif
4045
4046 #if !defined(__MACH__)
4047         info->stop_count = -1;
4048         info->signal = 0;
4049 #endif
4050         info->skip = 0;
4051         info->joined_stw = FALSE;
4052         info->doing_handshake = FALSE;
4053         info->thread_is_dying = FALSE;
4054         info->stack_start = NULL;
4055         info->stopped_ip = NULL;
4056         info->stopped_domain = NULL;
4057 #ifdef USE_MONO_CTX
4058         memset (&info->ctx, 0, sizeof (MonoContext));
4059 #else
4060         memset (&info->regs, 0, sizeof (info->regs));
4061 #endif
4062
4063         sgen_init_tlab_info (info);
4064
4065         binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
4066
4067         /* try to get it with attributes first */
4068 #if (defined(HAVE_PTHREAD_GETATTR_NP) || defined(HAVE_PTHREAD_ATTR_GET_NP)) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
4069   {
4070      size_t size;
4071      void *sstart;
4072      pthread_attr_t attr;
4073
4074 #if defined(HAVE_PTHREAD_GETATTR_NP)
4075     /* Linux */
4076     pthread_getattr_np (pthread_self (), &attr);
4077 #elif defined(HAVE_PTHREAD_ATTR_GET_NP)
4078     /* BSD */
4079     pthread_attr_init (&attr);
4080     pthread_attr_get_np (pthread_self (), &attr);
4081 #else
4082 #error Cannot determine which API is needed to retrieve pthread attributes.
4083 #endif
4084
4085      pthread_attr_getstack (&attr, &sstart, &size);
4086      info->stack_start_limit = sstart;
4087      info->stack_end = (char*)sstart + size;
4088      pthread_attr_destroy (&attr);
4089   }
4090 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
4091                  info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
4092                  info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
4093 #else
4094         {
4095                 /* FIXME: we assume the stack grows down */
4096                 gsize stack_bottom = (gsize)addr;
4097                 stack_bottom += 4095;
4098                 stack_bottom &= ~4095;
4099                 info->stack_end = (char*)stack_bottom;
4100         }
4101 #endif
4102
4103 #ifdef HAVE_KW_THREAD
4104         stack_end = info->stack_end;
4105 #endif
4106
4107         SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
4108
4109         if (gc_callbacks.thread_attach_func)
4110                 info->runtime_data = gc_callbacks.thread_attach_func ();
4111
4112         UNLOCK_GC;
4113         return info;
4114 }
4115
4116 static void
4117 sgen_thread_unregister (SgenThreadInfo *p)
4118 {
4119         /* If a delegate is passed to native code and invoked on a thread we dont
4120          * know about, the jit will register it with mono_jit_thread_attach, but
4121          * we have no way of knowing when that thread goes away.  SGen has a TSD
4122          * so we assume that if the domain is still registered, we can detach
4123          * the thread
4124          */
4125         if (mono_domain_get ())
4126                 mono_thread_detach (mono_thread_current ());
4127
4128         p->thread_is_dying = TRUE;
4129
4130         /*
4131         There is a race condition between a thread finishing executing and been removed
4132         from the GC thread set.
4133         This happens on posix systems when TLS data is been cleaned-up, libpthread will
4134         set the thread_info slot to NULL before calling the cleanup function. This
4135         opens a window in which the thread is registered but has a NULL TLS.
4136
4137         The suspend signal handler needs TLS data to know where to store thread state
4138         data or otherwise it will simply ignore the thread.
4139
4140         This solution works because the thread doing STW will wait until all threads been
4141         suspended handshake back, so there is no race between the doing_hankshake test
4142         and the suspend_thread call.
4143
4144         This is not required on systems that do synchronous STW as those can deal with
4145         the above race at suspend time.
4146
4147         FIXME: I believe we could avoid this by using mono_thread_info_lookup when
4148         mono_thread_info_current returns NULL. Or fix mono_thread_info_lookup to do so.
4149         */
4150 #if (defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED) || !defined(HAVE_PTHREAD_KILL)
4151         LOCK_GC;
4152 #else
4153         while (!TRYLOCK_GC) {
4154                 if (!sgen_park_current_thread_if_doing_handshake (p))
4155                         g_usleep (50);
4156         }
4157         MONO_GC_LOCKED ();
4158 #endif
4159
4160         binary_protocol_thread_unregister ((gpointer)mono_thread_info_get_tid (p));
4161         SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)mono_thread_info_get_tid (p));
4162
4163         if (gc_callbacks.thread_detach_func) {
4164                 gc_callbacks.thread_detach_func (p->runtime_data);
4165                 p->runtime_data = NULL;
4166         }
4167
4168         mono_threads_unregister_current_thread (p);
4169         UNLOCK_GC;
4170 }
4171
4172
4173 static void
4174 sgen_thread_attach (SgenThreadInfo *info)
4175 {
4176         LOCK_GC;
4177         /*this is odd, can we get attached before the gc is inited?*/
4178         init_stats ();
4179         UNLOCK_GC;
4180
4181         if (gc_callbacks.thread_attach_func && !info->runtime_data)
4182                 info->runtime_data = gc_callbacks.thread_attach_func ();
4183 }
4184 gboolean
4185 mono_gc_register_thread (void *baseptr)
4186 {
4187         return mono_thread_info_attach (baseptr) != NULL;
4188 }
4189
4190 /*
4191  * mono_gc_set_stack_end:
4192  *
4193  *   Set the end of the current threads stack to STACK_END. The stack space between
4194  * STACK_END and the real end of the threads stack will not be scanned during collections.
4195  */
4196 void
4197 mono_gc_set_stack_end (void *stack_end)
4198 {
4199         SgenThreadInfo *info;
4200
4201         LOCK_GC;
4202         info = mono_thread_info_current ();
4203         if (info) {
4204                 g_assert (stack_end < info->stack_end);
4205                 info->stack_end = stack_end;
4206         }
4207         UNLOCK_GC;
4208 }
4209
4210 #if USE_PTHREAD_INTERCEPT
4211
4212
4213 int
4214 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
4215 {
4216         return pthread_create (new_thread, attr, start_routine, arg);
4217 }
4218
4219 int
4220 mono_gc_pthread_join (pthread_t thread, void **retval)
4221 {
4222         return pthread_join (thread, retval);
4223 }
4224
4225 int
4226 mono_gc_pthread_detach (pthread_t thread)
4227 {
4228         return pthread_detach (thread);
4229 }
4230
4231 void
4232 mono_gc_pthread_exit (void *retval)
4233 {
4234         pthread_exit (retval);
4235 }
4236
4237 #endif /* USE_PTHREAD_INTERCEPT */
4238
4239 /*
4240  * ######################################################################
4241  * ########  Write barriers
4242  * ######################################################################
4243  */
4244
4245 /*
4246  * Note: the write barriers first do the needed GC work and then do the actual store:
4247  * this way the value is visible to the conservative GC scan after the write barrier
4248  * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
4249  * the conservative scan, otherwise by the remembered set scan.
4250  */
4251 void
4252 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
4253 {
4254         HEAVY_STAT (++stat_wbarrier_set_field);
4255         if (ptr_in_nursery (field_ptr)) {
4256                 *(void**)field_ptr = value;
4257                 return;
4258         }
4259         SGEN_LOG (8, "Adding remset at %p", field_ptr);
4260         if (value)
4261                 binary_protocol_wbarrier (field_ptr, value, value->vtable);
4262
4263         remset.wbarrier_set_field (obj, field_ptr, value);
4264 }
4265
4266 void
4267 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
4268 {
4269         HEAVY_STAT (++stat_wbarrier_set_arrayref);
4270         if (ptr_in_nursery (slot_ptr)) {
4271                 *(void**)slot_ptr = value;
4272                 return;
4273         }
4274         SGEN_LOG (8, "Adding remset at %p", slot_ptr);
4275         if (value)
4276                 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
4277
4278         remset.wbarrier_set_arrayref (arr, slot_ptr, value);
4279 }
4280
4281 void
4282 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
4283 {
4284         HEAVY_STAT (++stat_wbarrier_arrayref_copy);
4285         /*This check can be done without taking a lock since dest_ptr array is pinned*/
4286         if (ptr_in_nursery (dest_ptr) || count <= 0) {
4287                 mono_gc_memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
4288                 return;
4289         }
4290
4291 #ifdef SGEN_BINARY_PROTOCOL
4292         {
4293                 int i;
4294                 for (i = 0; i < count; ++i) {
4295                         gpointer dest = (gpointer*)dest_ptr + i;
4296                         gpointer obj = *((gpointer*)src_ptr + i);
4297                         if (obj)
4298                                 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
4299                 }
4300         }
4301 #endif
4302
4303         remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
4304 }
4305
4306 static char *found_obj;
4307
4308 static void
4309 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4310 {
4311         char *ptr = user_data;
4312
4313         if (ptr >= obj && ptr < obj + size) {
4314                 g_assert (!found_obj);
4315                 found_obj = obj;
4316         }
4317 }
4318
4319 /* for use in the debugger */
4320 char* find_object_for_ptr (char *ptr);
4321 char*
4322 find_object_for_ptr (char *ptr)
4323 {
4324         if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4325                 found_obj = NULL;
4326                 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4327                                 find_object_for_ptr_callback, ptr, TRUE);
4328                 if (found_obj)
4329                         return found_obj;
4330         }
4331
4332         found_obj = NULL;
4333         sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4334         if (found_obj)
4335                 return found_obj;
4336
4337         /*
4338          * Very inefficient, but this is debugging code, supposed to
4339          * be called from gdb, so we don't care.
4340          */
4341         found_obj = NULL;
4342         major_collector.iterate_objects (TRUE, TRUE, find_object_for_ptr_callback, ptr);
4343         return found_obj;
4344 }
4345
4346 void
4347 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4348 {
4349         gpointer obj;
4350
4351         HEAVY_STAT (++stat_wbarrier_generic_store);
4352
4353 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4354         /* FIXME: ptr_in_heap must be called with the GC lock held */
4355         if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4356                 char *start = find_object_for_ptr (ptr);
4357                 MonoObject *value = *(MonoObject**)ptr;
4358                 LOCK_GC;
4359                 g_assert (start);
4360                 if (start) {
4361                         MonoObject *obj = (MonoObject*)start;
4362                         if (obj->vtable->domain != value->vtable->domain)
4363                                 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4364                 }
4365                 UNLOCK_GC;
4366         }
4367 #endif
4368
4369         obj = *(gpointer*)ptr;
4370         if (obj)
4371                 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4372
4373         if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4374                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4375                 return;
4376         }
4377
4378         /*
4379          * We need to record old->old pointer locations for the
4380          * concurrent collector.
4381          */
4382         if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4383                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4384                 return;
4385         }
4386
4387         SGEN_LOG (8, "Adding remset at %p", ptr);
4388
4389         remset.wbarrier_generic_nostore (ptr);
4390 }
4391
4392 void
4393 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4394 {
4395         SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4396         *(void**)ptr = value;
4397         if (ptr_in_nursery (value))
4398                 mono_gc_wbarrier_generic_nostore (ptr);
4399         sgen_dummy_use (value);
4400 }
4401
4402 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4403 {
4404         mword *dest = _dest;
4405         mword *src = _src;
4406
4407         while (size) {
4408                 if (bitmap & 0x1)
4409                         mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4410                 else
4411                         *dest = *src;
4412                 ++src;
4413                 ++dest;
4414                 size -= SIZEOF_VOID_P;
4415                 bitmap >>= 1;
4416         }
4417 }
4418
4419 #ifdef SGEN_BINARY_PROTOCOL
4420 #undef HANDLE_PTR
4421 #define HANDLE_PTR(ptr,obj) do {                                        \
4422                 gpointer o = *(gpointer*)(ptr);                         \
4423                 if ((o)) {                                              \
4424                         gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4425                         binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4426                 }                                                       \
4427         } while (0)
4428
4429 static void
4430 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4431 {
4432 #define SCAN_OBJECT_NOVTABLE
4433 #include "sgen-scan-object.h"
4434 }
4435 #endif
4436
4437 void
4438 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4439 {
4440         HEAVY_STAT (++stat_wbarrier_value_copy);
4441         g_assert (klass->valuetype);
4442
4443         SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4444
4445         if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4446                 size_t element_size = mono_class_value_size (klass, NULL);
4447                 size_t size = count * element_size;
4448                 mono_gc_memmove (dest, src, size);
4449                 return;
4450         }
4451
4452 #ifdef SGEN_BINARY_PROTOCOL
4453         {
4454                 size_t element_size = mono_class_value_size (klass, NULL);
4455                 int i;
4456                 for (i = 0; i < count; ++i) {
4457                         scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4458                                         (char*)src + i * element_size - sizeof (MonoObject),
4459                                         (mword) klass->gc_descr);
4460                 }
4461         }
4462 #endif
4463
4464         remset.wbarrier_value_copy (dest, src, count, klass);
4465 }
4466
4467 /**
4468  * mono_gc_wbarrier_object_copy:
4469  *
4470  * Write barrier to call when obj is the result of a clone or copy of an object.
4471  */
4472 void
4473 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4474 {
4475         int size;
4476
4477         HEAVY_STAT (++stat_wbarrier_object_copy);
4478
4479         if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4480                 size = mono_object_class (obj)->instance_size;
4481                 mono_gc_memmove ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4482                                 size - sizeof (MonoObject));
4483                 return;
4484         }
4485
4486 #ifdef SGEN_BINARY_PROTOCOL
4487         scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4488 #endif
4489
4490         remset.wbarrier_object_copy (obj, src);
4491 }
4492
4493
4494 /*
4495  * ######################################################################
4496  * ########  Other mono public interface functions.
4497  * ######################################################################
4498  */
4499
4500 #define REFS_SIZE 128
4501 typedef struct {
4502         void *data;
4503         MonoGCReferences callback;
4504         int flags;
4505         int count;
4506         int called;
4507         MonoObject *refs [REFS_SIZE];
4508         uintptr_t offsets [REFS_SIZE];
4509 } HeapWalkInfo;
4510
4511 #undef HANDLE_PTR
4512 #define HANDLE_PTR(ptr,obj)     do {    \
4513                 if (*(ptr)) {   \
4514                         if (hwi->count == REFS_SIZE) {  \
4515                                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);    \
4516                                 hwi->count = 0; \
4517                                 hwi->called = 1;        \
4518                         }       \
4519                         hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start;  \
4520                         hwi->refs [hwi->count++] = *(ptr);      \
4521                 }       \
4522         } while (0)
4523
4524 static void
4525 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4526 {
4527 #include "sgen-scan-object.h"
4528 }
4529
4530 static void
4531 walk_references (char *start, size_t size, void *data)
4532 {
4533         HeapWalkInfo *hwi = data;
4534         hwi->called = 0;
4535         hwi->count = 0;
4536         collect_references (hwi, start, size);
4537         if (hwi->count || !hwi->called)
4538                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4539 }
4540
4541 /**
4542  * mono_gc_walk_heap:
4543  * @flags: flags for future use
4544  * @callback: a function pointer called for each object in the heap
4545  * @data: a user data pointer that is passed to callback
4546  *
4547  * This function can be used to iterate over all the live objects in the heap:
4548  * for each object, @callback is invoked, providing info about the object's
4549  * location in memory, its class, its size and the objects it references.
4550  * For each referenced object it's offset from the object address is
4551  * reported in the offsets array.
4552  * The object references may be buffered, so the callback may be invoked
4553  * multiple times for the same object: in all but the first call, the size
4554  * argument will be zero.
4555  * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4556  * profiler event handler.
4557  *
4558  * Returns: a non-zero value if the GC doesn't support heap walking
4559  */
4560 int
4561 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4562 {
4563         HeapWalkInfo hwi;
4564
4565         hwi.flags = flags;
4566         hwi.callback = callback;
4567         hwi.data = data;
4568
4569         sgen_clear_nursery_fragments ();
4570         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4571
4572         major_collector.iterate_objects (TRUE, TRUE, walk_references, &hwi);
4573         sgen_los_iterate_objects (walk_references, &hwi);
4574
4575         return 0;
4576 }
4577
4578 void
4579 mono_gc_collect (int generation)
4580 {
4581         LOCK_GC;
4582         if (generation > 1)
4583                 generation = 1;
4584         sgen_perform_collection (0, generation, "user request", TRUE);
4585         UNLOCK_GC;
4586 }
4587
4588 int
4589 mono_gc_max_generation (void)
4590 {
4591         return 1;
4592 }
4593
4594 int
4595 mono_gc_collection_count (int generation)
4596 {
4597         if (generation == 0)
4598                 return stat_minor_gcs;
4599         return stat_major_gcs;
4600 }
4601
4602 int64_t
4603 mono_gc_get_used_size (void)
4604 {
4605         gint64 tot = 0;
4606         LOCK_GC;
4607         tot = los_memory_usage;
4608         tot += nursery_section->next_data - nursery_section->data;
4609         tot += major_collector.get_used_size ();
4610         /* FIXME: account for pinned objects */
4611         UNLOCK_GC;
4612         return tot;
4613 }
4614
4615 int
4616 mono_gc_get_los_limit (void)
4617 {
4618         return MAX_SMALL_OBJ_SIZE;
4619 }
4620
4621 gboolean
4622 mono_gc_user_markers_supported (void)
4623 {
4624         return TRUE;
4625 }
4626
4627 gboolean
4628 mono_object_is_alive (MonoObject* o)
4629 {
4630         return TRUE;
4631 }
4632
4633 int
4634 mono_gc_get_generation (MonoObject *obj)
4635 {
4636         if (ptr_in_nursery (obj))
4637                 return 0;
4638         return 1;
4639 }
4640
4641 void
4642 mono_gc_enable_events (void)
4643 {
4644 }
4645
4646 void
4647 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4648 {
4649         sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4650 }
4651
4652 void
4653 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4654 {
4655         sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4656 }
4657
4658 MonoObject*
4659 mono_gc_weak_link_get (void **link_addr)
4660 {
4661         void * volatile *link_addr_volatile;
4662         void *ptr;
4663         MonoObject *obj;
4664  retry:
4665         link_addr_volatile = link_addr;
4666         ptr = (void*)*link_addr_volatile;
4667         /*
4668          * At this point we have a hidden pointer.  If the GC runs
4669          * here, it will not recognize the hidden pointer as a
4670          * reference, and if the object behind it is not referenced
4671          * elsewhere, it will be freed.  Once the world is restarted
4672          * we reveal the pointer, giving us a pointer to a freed
4673          * object.  To make sure we don't return it, we load the
4674          * hidden pointer again.  If it's still the same, we can be
4675          * sure the object reference is valid.
4676          */
4677         if (ptr)
4678                 obj = (MonoObject*) REVEAL_POINTER (ptr);
4679         else
4680                 return NULL;
4681
4682         mono_memory_barrier ();
4683
4684         /*
4685          * During the second bridge processing step the world is
4686          * running again.  That step processes all weak links once
4687          * more to null those that refer to dead objects.  Before that
4688          * is completed, those links must not be followed, so we
4689          * conservatively wait for bridge processing when any weak
4690          * link is dereferenced.
4691          */
4692         if (G_UNLIKELY (bridge_processing_in_progress))
4693                 mono_gc_wait_for_bridge_processing ();
4694
4695         if ((void*)*link_addr_volatile != ptr)
4696                 goto retry;
4697
4698         return obj;
4699 }
4700
4701 gboolean
4702 mono_gc_ephemeron_array_add (MonoObject *obj)
4703 {
4704         EphemeronLinkNode *node;
4705
4706         LOCK_GC;
4707
4708         node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4709         if (!node) {
4710                 UNLOCK_GC;
4711                 return FALSE;
4712         }
4713         node->array = (char*)obj;
4714         node->next = ephemeron_list;
4715         ephemeron_list = node;
4716
4717         SGEN_LOG (5, "Registered ephemeron array %p", obj);
4718
4719         UNLOCK_GC;
4720         return TRUE;
4721 }
4722
4723 gboolean
4724 mono_gc_set_allow_synchronous_major (gboolean flag)
4725 {
4726         if (!major_collector.is_concurrent)
4727                 return flag;
4728
4729         allow_synchronous_major = flag;
4730         return TRUE;
4731 }
4732
4733 void*
4734 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4735 {
4736         void *result;
4737         LOCK_INTERRUPTION;
4738         result = func (data);
4739         UNLOCK_INTERRUPTION;
4740         return result;
4741 }
4742
4743 gboolean
4744 mono_gc_is_gc_thread (void)
4745 {
4746         gboolean result;
4747         LOCK_GC;
4748         result = mono_thread_info_current () != NULL;
4749         UNLOCK_GC;
4750         return result;
4751 }
4752
4753 static gboolean
4754 is_critical_method (MonoMethod *method)
4755 {
4756         return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4757 }
4758
4759 void
4760 sgen_env_var_error (const char *env_var, const char *fallback, const char *description_format, ...)
4761 {
4762         va_list ap;
4763
4764         va_start (ap, description_format);
4765
4766         fprintf (stderr, "Warning: In environment variable `%s': ", env_var);
4767         vfprintf (stderr, description_format, ap);
4768         if (fallback)
4769                 fprintf (stderr, " - %s", fallback);
4770         fprintf (stderr, "\n");
4771
4772         va_end (ap);
4773 }
4774
4775 static gboolean
4776 parse_double_in_interval (const char *env_var, const char *opt_name, const char *opt, double min, double max, double *result)
4777 {
4778         char *endptr;
4779         double val = strtod (opt, &endptr);
4780         if (endptr == opt) {
4781                 sgen_env_var_error (env_var, "Using default value.", "`%s` must be a number.", opt_name);
4782                 return FALSE;
4783         }
4784         else if (val < min || val > max) {
4785                 sgen_env_var_error (env_var, "Using default value.", "`%s` must be between %.2f - %.2f.", opt_name, min, max);
4786                 return FALSE;
4787         }
4788         *result = val;
4789         return TRUE;
4790 }
4791
4792 void
4793 mono_gc_base_init (void)
4794 {
4795         MonoThreadInfoCallbacks cb;
4796         char *env;
4797         char **opts, **ptr;
4798         char *major_collector_opt = NULL;
4799         char *minor_collector_opt = NULL;
4800         glong max_heap = 0;
4801         glong soft_limit = 0;
4802         int num_workers;
4803         int result;
4804         int dummy;
4805         gboolean debug_print_allowance = FALSE;
4806         double allowance_ratio = 0, save_target = 0;
4807         gboolean have_split_nursery = FALSE;
4808         gboolean cement_enabled = TRUE;
4809
4810         do {
4811                 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4812                 switch (result) {
4813                 case 1:
4814                         /* already inited */
4815                         return;
4816                 case -1:
4817                         /* being inited by another thread */
4818                         g_usleep (1000);
4819                         break;
4820                 case 0:
4821                         /* we will init it */
4822                         break;
4823                 default:
4824                         g_assert_not_reached ();
4825                 }
4826         } while (result != 0);
4827
4828         LOCK_INIT (gc_mutex);
4829
4830         pagesize = mono_pagesize ();
4831         gc_debug_file = stderr;
4832
4833         cb.thread_register = sgen_thread_register;
4834         cb.thread_unregister = sgen_thread_unregister;
4835         cb.thread_attach = sgen_thread_attach;
4836         cb.mono_method_is_critical = (gpointer)is_critical_method;
4837 #ifndef HOST_WIN32
4838         cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4839 #endif
4840
4841         mono_threads_init (&cb, sizeof (SgenThreadInfo));
4842
4843         LOCK_INIT (sgen_interruption_mutex);
4844         LOCK_INIT (pin_queue_mutex);
4845
4846         init_user_copy_or_mark_key ();
4847
4848         if ((env = getenv (MONO_GC_PARAMS_NAME))) {
4849                 opts = g_strsplit (env, ",", -1);
4850                 for (ptr = opts; *ptr; ++ptr) {
4851                         char *opt = *ptr;
4852                         if (g_str_has_prefix (opt, "major=")) {
4853                                 opt = strchr (opt, '=') + 1;
4854                                 major_collector_opt = g_strdup (opt);
4855                         } else if (g_str_has_prefix (opt, "minor=")) {
4856                                 opt = strchr (opt, '=') + 1;
4857                                 minor_collector_opt = g_strdup (opt);
4858                         }
4859                 }
4860         } else {
4861                 opts = NULL;
4862         }
4863
4864         init_stats ();
4865         sgen_init_internal_allocator ();
4866         sgen_init_nursery_allocator ();
4867
4868         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4869         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4870         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4871         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4872
4873 #ifndef HAVE_KW_THREAD
4874         mono_native_tls_alloc (&thread_info_key, NULL);
4875 #endif
4876
4877         /*
4878          * This needs to happen before any internal allocations because
4879          * it inits the small id which is required for hazard pointer
4880          * operations.
4881          */
4882         sgen_os_init ();
4883
4884         mono_thread_info_attach (&dummy);
4885
4886         if (!minor_collector_opt) {
4887                 sgen_simple_nursery_init (&sgen_minor_collector);
4888         } else {
4889                 if (!strcmp (minor_collector_opt, "simple")) {
4890                 use_simple_nursery:
4891                         sgen_simple_nursery_init (&sgen_minor_collector);
4892                 } else if (!strcmp (minor_collector_opt, "split")) {
4893                         sgen_split_nursery_init (&sgen_minor_collector);
4894                         have_split_nursery = TRUE;
4895                 } else {
4896                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `simple` instead.", "Unknown minor collector `%s'.", minor_collector_opt);
4897                         goto use_simple_nursery;
4898                 }
4899         }
4900
4901         if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4902         use_marksweep_major:
4903                 sgen_marksweep_init (&major_collector);
4904         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed")) {
4905                 sgen_marksweep_fixed_init (&major_collector);
4906         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-par")) {
4907                 sgen_marksweep_par_init (&major_collector);
4908         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) {
4909                 sgen_marksweep_fixed_par_init (&major_collector);
4910         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4911                 sgen_marksweep_conc_init (&major_collector);
4912         } else {
4913                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `marksweep` instead.", "Unknown major collector `%s'.", major_collector_opt);
4914                 goto use_marksweep_major;
4915         }
4916
4917         if (have_split_nursery && major_collector.is_parallel) {
4918                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Disabling split minor collector.", "`minor=split` is not supported with the parallel collector yet.");
4919                 have_split_nursery = FALSE;
4920         }
4921
4922         num_workers = mono_cpu_count ();
4923         g_assert (num_workers > 0);
4924         if (num_workers > 16)
4925                 num_workers = 16;
4926
4927         ///* Keep this the default for now */
4928         /* Precise marking is broken on all supported targets. Disable until fixed. */
4929         conservative_stack_mark = TRUE;
4930
4931         sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4932
4933         if (opts) {
4934                 gboolean usage_printed = FALSE;
4935
4936                 for (ptr = opts; *ptr; ++ptr) {
4937                         char *opt = *ptr;
4938                         if (!strcmp (opt, ""))
4939                                 continue;
4940                         if (g_str_has_prefix (opt, "major="))
4941                                 continue;
4942                         if (g_str_has_prefix (opt, "minor="))
4943                                 continue;
4944                         if (g_str_has_prefix (opt, "max-heap-size=")) {
4945                                 glong max_heap_candidate = 0;
4946                                 opt = strchr (opt, '=') + 1;
4947                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap_candidate)) {
4948                                         max_heap = (max_heap_candidate + mono_pagesize () - 1) & ~(glong)(mono_pagesize () - 1);
4949                                         if (max_heap != max_heap_candidate)
4950                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Rounding up.", "`max-heap-size` size must be a multiple of %d.", mono_pagesize ());
4951                                 } else {
4952                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`max-heap-size` must be an integer.");
4953                                 }
4954                                 continue;
4955                         }
4956                         if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4957                                 opt = strchr (opt, '=') + 1;
4958                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4959                                         if (soft_limit <= 0) {
4960                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be positive.");
4961                                                 soft_limit = 0;
4962                                         }
4963                                 } else {
4964                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be an integer.");
4965                                 }
4966                                 continue;
4967                         }
4968                         if (g_str_has_prefix (opt, "workers=")) {
4969                                 long val;
4970                                 char *endptr;
4971                                 if (!major_collector.is_parallel) {
4972                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "The `workers` option can only be used for parallel collectors.");
4973                                         continue;
4974                                 }
4975                                 opt = strchr (opt, '=') + 1;
4976                                 val = strtol (opt, &endptr, 10);
4977                                 if (!*opt || *endptr) {
4978                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Cannot parse the `workers` option value.");
4979                                         continue;
4980                                 }
4981                                 if (val <= 0 || val > 16) {
4982                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "The number of `workers` must be in the range 1 to 16.");
4983                                         continue;
4984                                 }
4985                                 num_workers = (int)val;
4986                                 continue;
4987                         }
4988                         if (g_str_has_prefix (opt, "stack-mark=")) {
4989                                 opt = strchr (opt, '=') + 1;
4990                                 if (!strcmp (opt, "precise")) {
4991                                         conservative_stack_mark = FALSE;
4992                                 } else if (!strcmp (opt, "conservative")) {
4993                                         conservative_stack_mark = TRUE;
4994                                 } else {
4995                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, conservative_stack_mark ? "Using `conservative`." : "Using `precise`.",
4996                                                         "Invalid value `%s` for `stack-mark` option, possible values are: `precise`, `conservative`.", opt);
4997                                 }
4998                                 continue;
4999                         }
5000                         if (g_str_has_prefix (opt, "bridge=")) {
5001                                 opt = strchr (opt, '=') + 1;
5002                                 sgen_register_test_bridge_callbacks (g_strdup (opt));
5003                                 continue;
5004                         }
5005 #ifdef USER_CONFIG
5006                         if (g_str_has_prefix (opt, "nursery-size=")) {
5007                                 long val;
5008                                 opt = strchr (opt, '=') + 1;
5009                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
5010 #ifdef SGEN_ALIGN_NURSERY
5011                                         if ((val & (val - 1))) {
5012                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be a power of two.");
5013                                                 continue;
5014                                         }
5015
5016                                         if (val < SGEN_MAX_NURSERY_WASTE) {
5017                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.",
5018                                                                 "`nursery-size` must be at least %d bytes.\n", SGEN_MAX_NURSERY_WASTE);
5019                                                 continue;
5020                                         }
5021
5022                                         sgen_nursery_size = val;
5023                                         sgen_nursery_bits = 0;
5024                                         while (1 << (++ sgen_nursery_bits) != sgen_nursery_size)
5025                                                 ;
5026 #else
5027                                         sgen_nursery_size = val;
5028 #endif
5029                                 } else {
5030                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be an integer.");
5031                                         continue;
5032                                 }
5033                                 continue;
5034                         }
5035 #endif
5036                         if (g_str_has_prefix (opt, "save-target-ratio=")) {
5037                                 double val;
5038                                 opt = strchr (opt, '=') + 1;
5039                                 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "save-target-ratio", opt,
5040                                                 SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO, &val)) {
5041                                         save_target = val;
5042                                 }
5043                                 continue;
5044                         }
5045                         if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
5046                                 double val;
5047                                 opt = strchr (opt, '=') + 1;
5048                                 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "default-allowance-ratio", opt,
5049                                                 SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, &val)) {
5050                                         allowance_ratio = val;
5051                                 }
5052                                 continue;
5053                         }
5054                         if (g_str_has_prefix (opt, "allow-synchronous-major=")) {
5055                                 if (!major_collector.is_concurrent) {
5056                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "`allow-synchronous-major` is only valid for the concurrent major collector.");
5057                                         continue;
5058                                 }
5059
5060                                 opt = strchr (opt, '=') + 1;
5061
5062                                 if (!strcmp (opt, "yes")) {
5063                                         allow_synchronous_major = TRUE;
5064                                 } else if (!strcmp (opt, "no")) {
5065                                         allow_synchronous_major = FALSE;
5066                                 } else {
5067                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`allow-synchronous-major` must be either `yes' or `no'.");
5068                                         continue;
5069                                 }
5070                         }
5071
5072                         if (!strcmp (opt, "cementing")) {
5073                                 cement_enabled = TRUE;
5074                                 continue;
5075                         }
5076                         if (!strcmp (opt, "no-cementing")) {
5077                                 cement_enabled = FALSE;
5078                                 continue;
5079                         }
5080
5081                         if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
5082                                 continue;
5083
5084                         if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
5085                                 continue;
5086
5087                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Unknown option `%s`.", opt);
5088
5089                         if (usage_printed)
5090                                 continue;
5091
5092                         fprintf (stderr, "\n%s must be a comma-delimited list of one or more of the following:\n", MONO_GC_PARAMS_NAME);
5093                         fprintf (stderr, "  max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5094                         fprintf (stderr, "  soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
5095                         fprintf (stderr, "  nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5096                         fprintf (stderr, "  major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-conc', `marksweep-par', 'marksweep-fixed' or 'marksweep-fixed-par')\n");
5097                         fprintf (stderr, "  minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
5098                         fprintf (stderr, "  wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
5099                         fprintf (stderr, "  stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
5100                         fprintf (stderr, "  [no-]cementing\n");
5101                         if (major_collector.is_concurrent)
5102                                 fprintf (stderr, "  allow-synchronous-major=FLAG (where FLAG is `yes' or `no')\n");
5103                         if (major_collector.print_gc_param_usage)
5104                                 major_collector.print_gc_param_usage ();
5105                         if (sgen_minor_collector.print_gc_param_usage)
5106                                 sgen_minor_collector.print_gc_param_usage ();
5107                         fprintf (stderr, " Experimental options:\n");
5108                         fprintf (stderr, "  save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
5109                         fprintf (stderr, "  default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
5110                         fprintf (stderr, "\n");
5111
5112                         usage_printed = TRUE;
5113                 }
5114                 g_strfreev (opts);
5115         }
5116
5117         if (major_collector.is_parallel)
5118                 sgen_workers_init (num_workers);
5119         else if (major_collector.is_concurrent)
5120                 sgen_workers_init (1);
5121
5122         if (major_collector_opt)
5123                 g_free (major_collector_opt);
5124
5125         if (minor_collector_opt)
5126                 g_free (minor_collector_opt);
5127
5128         alloc_nursery ();
5129
5130         sgen_cement_init (cement_enabled);
5131
5132         if ((env = getenv (MONO_GC_DEBUG_NAME))) {
5133                 gboolean usage_printed = FALSE;
5134
5135                 opts = g_strsplit (env, ",", -1);
5136                 for (ptr = opts; ptr && *ptr; ptr ++) {
5137                         char *opt = *ptr;
5138                         if (!strcmp (opt, ""))
5139                                 continue;
5140                         if (opt [0] >= '0' && opt [0] <= '9') {
5141                                 gc_debug_level = atoi (opt);
5142                                 opt++;
5143                                 if (opt [0] == ':')
5144                                         opt++;
5145                                 if (opt [0]) {
5146 #ifdef HOST_WIN32
5147                                         char *rf = g_strdup_printf ("%s.%d", opt, GetCurrentProcessId ());
5148 #else
5149                                         char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
5150 #endif
5151                                         gc_debug_file = fopen (rf, "wb");
5152                                         if (!gc_debug_file)
5153                                                 gc_debug_file = stderr;
5154                                         g_free (rf);
5155                                 }
5156                         } else if (!strcmp (opt, "print-allowance")) {
5157                                 debug_print_allowance = TRUE;
5158                         } else if (!strcmp (opt, "print-pinning")) {
5159                                 do_pin_stats = TRUE;
5160                         } else if (!strcmp (opt, "verify-before-allocs")) {
5161                                 verify_before_allocs = 1;
5162                                 has_per_allocation_action = TRUE;
5163                         } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
5164                                 char *arg = strchr (opt, '=') + 1;
5165                                 verify_before_allocs = atoi (arg);
5166                                 has_per_allocation_action = TRUE;
5167                         } else if (!strcmp (opt, "collect-before-allocs")) {
5168                                 collect_before_allocs = 1;
5169                                 has_per_allocation_action = TRUE;
5170                         } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
5171                                 char *arg = strchr (opt, '=') + 1;
5172                                 has_per_allocation_action = TRUE;
5173                                 collect_before_allocs = atoi (arg);
5174                         } else if (!strcmp (opt, "verify-before-collections")) {
5175                                 whole_heap_check_before_collection = TRUE;
5176                         } else if (!strcmp (opt, "check-at-minor-collections")) {
5177                                 consistency_check_at_minor_collection = TRUE;
5178                                 nursery_clear_policy = CLEAR_AT_GC;
5179                         } else if (!strcmp (opt, "mod-union-consistency-check")) {
5180                                 if (!major_collector.is_concurrent) {
5181                                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`mod-union-consistency-check` only works with concurrent major collector.");
5182                                         continue;
5183                                 }
5184                                 mod_union_consistency_check = TRUE;
5185                         } else if (!strcmp (opt, "check-mark-bits")) {
5186                                 check_mark_bits_after_major_collection = TRUE;
5187                         } else if (!strcmp (opt, "check-nursery-pinned")) {
5188                                 check_nursery_objects_pinned = TRUE;
5189                         } else if (!strcmp (opt, "xdomain-checks")) {
5190                                 xdomain_checks = TRUE;
5191                         } else if (!strcmp (opt, "clear-at-gc")) {
5192                                 nursery_clear_policy = CLEAR_AT_GC;
5193                         } else if (!strcmp (opt, "clear-nursery-at-gc")) {
5194                                 nursery_clear_policy = CLEAR_AT_GC;
5195                         } else if (!strcmp (opt, "check-scan-starts")) {
5196                                 do_scan_starts_check = TRUE;
5197                         } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
5198                                 do_verify_nursery = TRUE;
5199                         } else if (!strcmp (opt, "check-concurrent")) {
5200                                 if (!major_collector.is_concurrent) {
5201                                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`check-concurrent` only works with concurrent major collectors.");
5202                                         continue;
5203                                 }
5204                                 do_concurrent_checks = TRUE;
5205                         } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
5206                                 do_dump_nursery_content = TRUE;
5207                         } else if (!strcmp (opt, "no-managed-allocator")) {
5208                                 sgen_set_use_managed_allocator (FALSE);
5209                         } else if (!strcmp (opt, "disable-minor")) {
5210                                 disable_minor_collections = TRUE;
5211                         } else if (!strcmp (opt, "disable-major")) {
5212                                 disable_major_collections = TRUE;
5213                         } else if (g_str_has_prefix (opt, "heap-dump=")) {
5214                                 char *filename = strchr (opt, '=') + 1;
5215                                 nursery_clear_policy = CLEAR_AT_GC;
5216                                 heap_dump_file = fopen (filename, "w");
5217                                 if (heap_dump_file) {
5218                                         fprintf (heap_dump_file, "<sgen-dump>\n");
5219                                         do_pin_stats = TRUE;
5220                                 }
5221 #ifdef SGEN_BINARY_PROTOCOL
5222                         } else if (g_str_has_prefix (opt, "binary-protocol=")) {
5223                                 char *filename = strchr (opt, '=') + 1;
5224                                 binary_protocol_init (filename);
5225 #endif
5226                         } else {
5227                                 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "Unknown option `%s`.", opt);
5228
5229                                 if (usage_printed)
5230                                         continue;
5231
5232                                 fprintf (stderr, "\n%s must be of the format [<l>[:<filename>]|<option>]+ where <l> is a debug level 0-9.\n", MONO_GC_DEBUG_NAME);
5233                                 fprintf (stderr, "Valid <option>s are:\n");
5234                                 fprintf (stderr, "  collect-before-allocs[=<n>]\n");
5235                                 fprintf (stderr, "  verify-before-allocs[=<n>]\n");
5236                                 fprintf (stderr, "  check-at-minor-collections\n");
5237                                 fprintf (stderr, "  check-mark-bits\n");
5238                                 fprintf (stderr, "  check-nursery-pinned\n");
5239                                 fprintf (stderr, "  verify-before-collections\n");
5240                                 fprintf (stderr, "  verify-nursery-at-minor-gc\n");
5241                                 fprintf (stderr, "  dump-nursery-at-minor-gc\n");
5242                                 fprintf (stderr, "  disable-minor\n");
5243                                 fprintf (stderr, "  disable-major\n");
5244                                 fprintf (stderr, "  xdomain-checks\n");
5245                                 fprintf (stderr, "  check-concurrent\n");
5246                                 fprintf (stderr, "  clear-at-gc\n");
5247                                 fprintf (stderr, "  clear-nursery-at-gc\n");
5248                                 fprintf (stderr, "  check-scan-starts\n");
5249                                 fprintf (stderr, "  no-managed-allocator\n");
5250                                 fprintf (stderr, "  print-allowance\n");
5251                                 fprintf (stderr, "  print-pinning\n");
5252                                 fprintf (stderr, "  heap-dump=<filename>\n");
5253 #ifdef SGEN_BINARY_PROTOCOL
5254                                 fprintf (stderr, "  binary-protocol=<filename>\n");
5255 #endif
5256                                 fprintf (stderr, "\n");
5257
5258                                 usage_printed = TRUE;
5259                         }
5260                 }
5261                 g_strfreev (opts);
5262         }
5263
5264         if (major_collector.is_parallel) {
5265                 if (heap_dump_file) {
5266                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Disabling.", "Cannot do `heap-dump` with the parallel collector.");
5267                         fclose (heap_dump_file);
5268                         heap_dump_file = NULL;
5269                 }
5270                 if (do_pin_stats) {
5271                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Disabling.", "`print-pinning` is not supported with the parallel collector.");
5272                         do_pin_stats = FALSE;
5273                 }
5274         }
5275
5276         if (major_collector.post_param_init)
5277                 major_collector.post_param_init (&major_collector);
5278
5279         sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
5280
5281         memset (&remset, 0, sizeof (remset));
5282
5283         sgen_card_table_init (&remset);
5284
5285         gc_initialized = 1;
5286 }
5287
5288 const char *
5289 mono_gc_get_gc_name (void)
5290 {
5291         return "sgen";
5292 }
5293
5294 static MonoMethod *write_barrier_method;
5295
5296 gboolean
5297 sgen_is_critical_method (MonoMethod *method)
5298 {
5299         return (method == write_barrier_method || sgen_is_managed_allocator (method));
5300 }
5301
5302 gboolean
5303 sgen_has_critical_method (void)
5304 {
5305         return write_barrier_method || sgen_has_managed_allocator ();
5306 }
5307
5308 #ifndef DISABLE_JIT
5309
5310 static void
5311 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5312 {
5313         memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5314 #ifdef SGEN_ALIGN_NURSERY
5315         // if (ptr_in_nursery (ptr)) return;
5316         /*
5317          * Masking out the bits might be faster, but we would have to use 64 bit
5318          * immediates, which might be slower.
5319          */
5320         mono_mb_emit_ldarg (mb, 0);
5321         mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5322         mono_mb_emit_byte (mb, CEE_SHR_UN);
5323         mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5324         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5325
5326         if (!major_collector.is_concurrent) {
5327                 // if (!ptr_in_nursery (*ptr)) return;
5328                 mono_mb_emit_ldarg (mb, 0);
5329                 mono_mb_emit_byte (mb, CEE_LDIND_I);
5330                 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5331                 mono_mb_emit_byte (mb, CEE_SHR_UN);
5332                 mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5333                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5334         }
5335 #else
5336         int label_continue1, label_continue2;
5337         int dereferenced_var;
5338
5339         // if (ptr < (sgen_get_nursery_start ())) goto continue;
5340         mono_mb_emit_ldarg (mb, 0);
5341         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5342         label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5343
5344         // if (ptr >= sgen_get_nursery_end ())) goto continue;
5345         mono_mb_emit_ldarg (mb, 0);
5346         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5347         label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5348
5349         // Otherwise return
5350         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5351
5352         // continue:
5353         mono_mb_patch_branch (mb, label_continue_1);
5354         mono_mb_patch_branch (mb, label_continue_2);
5355
5356         // Dereference and store in local var
5357         dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5358         mono_mb_emit_ldarg (mb, 0);
5359         mono_mb_emit_byte (mb, CEE_LDIND_I);
5360         mono_mb_emit_stloc (mb, dereferenced_var);
5361
5362         if (!major_collector.is_concurrent) {
5363                 // if (*ptr < sgen_get_nursery_start ()) return;
5364                 mono_mb_emit_ldloc (mb, dereferenced_var);
5365                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5366                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5367
5368                 // if (*ptr >= sgen_get_nursery_end ()) return;
5369                 mono_mb_emit_ldloc (mb, dereferenced_var);
5370                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5371                 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5372         }
5373 #endif
5374 }
5375 #endif
5376
5377 MonoMethod*
5378 mono_gc_get_write_barrier (void)
5379 {
5380         MonoMethod *res;
5381         MonoMethodBuilder *mb;
5382         MonoMethodSignature *sig;
5383 #ifdef MANAGED_WBARRIER
5384         int i, nursery_check_labels [3];
5385
5386 #ifdef HAVE_KW_THREAD
5387         int stack_end_offset = -1;
5388
5389         MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5390         g_assert (stack_end_offset != -1);
5391 #endif
5392 #endif
5393
5394         // FIXME: Maybe create a separate version for ctors (the branch would be
5395         // correctly predicted more times)
5396         if (write_barrier_method)
5397                 return write_barrier_method;
5398
5399         /* Create the IL version of mono_gc_barrier_generic_store () */
5400         sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5401         sig->ret = &mono_defaults.void_class->byval_arg;
5402         sig->params [0] = &mono_defaults.int_class->byval_arg;
5403
5404         mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5405
5406 #ifndef DISABLE_JIT
5407 #ifdef MANAGED_WBARRIER
5408         emit_nursery_check (mb, nursery_check_labels);
5409         /*
5410         addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5411         *addr = 1;
5412
5413         sgen_cardtable:
5414                 LDC_PTR sgen_cardtable
5415
5416         address >> CARD_BITS
5417                 LDARG_0
5418                 LDC_I4 CARD_BITS
5419                 SHR_UN
5420         if (SGEN_HAVE_OVERLAPPING_CARDS) {
5421                 LDC_PTR card_table_mask
5422                 AND
5423         }
5424         AND
5425         ldc_i4_1
5426         stind_i1
5427         */
5428         mono_mb_emit_ptr (mb, sgen_cardtable);
5429         mono_mb_emit_ldarg (mb, 0);
5430         mono_mb_emit_icon (mb, CARD_BITS);
5431         mono_mb_emit_byte (mb, CEE_SHR_UN);
5432 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5433         mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5434         mono_mb_emit_byte (mb, CEE_AND);
5435 #endif
5436         mono_mb_emit_byte (mb, CEE_ADD);
5437         mono_mb_emit_icon (mb, 1);
5438         mono_mb_emit_byte (mb, CEE_STIND_I1);
5439
5440         // return;
5441         for (i = 0; i < 3; ++i) {
5442                 if (nursery_check_labels [i])
5443                         mono_mb_patch_branch (mb, nursery_check_labels [i]);
5444         }
5445         mono_mb_emit_byte (mb, CEE_RET);
5446 #else
5447         mono_mb_emit_ldarg (mb, 0);
5448         mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5449         mono_mb_emit_byte (mb, CEE_RET);
5450 #endif
5451 #endif
5452         res = mono_mb_create_method (mb, sig, 16);
5453         mono_mb_free (mb);
5454
5455         mono_loader_lock ();
5456         if (write_barrier_method) {
5457                 /* Already created */
5458                 mono_free_method (res);
5459         } else {
5460                 /* double-checked locking */
5461                 mono_memory_barrier ();
5462                 write_barrier_method = res;
5463         }
5464         mono_loader_unlock ();
5465
5466         return write_barrier_method;
5467 }
5468
5469 char*
5470 mono_gc_get_description (void)
5471 {
5472         return g_strdup ("sgen");
5473 }
5474
5475 void
5476 mono_gc_set_desktop_mode (void)
5477 {
5478 }
5479
5480 gboolean
5481 mono_gc_is_moving (void)
5482 {
5483         return TRUE;
5484 }
5485
5486 gboolean
5487 mono_gc_is_disabled (void)
5488 {
5489         return FALSE;
5490 }
5491
5492 #ifdef HOST_WIN32
5493 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5494 {
5495         return TRUE;
5496 }
5497 #endif
5498
5499 NurseryClearPolicy
5500 sgen_get_nursery_clear_policy (void)
5501 {
5502         return nursery_clear_policy;
5503 }
5504
5505 MonoVTable*
5506 sgen_get_array_fill_vtable (void)
5507 {
5508         if (!array_fill_vtable) {
5509                 static MonoClass klass;
5510                 static MonoVTable vtable;
5511                 gsize bmap;
5512
5513                 MonoDomain *domain = mono_get_root_domain ();
5514                 g_assert (domain);
5515
5516                 klass.element_class = mono_defaults.byte_class;
5517                 klass.rank = 1;
5518                 klass.instance_size = sizeof (MonoArray);
5519                 klass.sizes.element_size = 1;
5520                 klass.name = "array_filler_type";
5521
5522                 vtable.klass = &klass;
5523                 bmap = 0;
5524                 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5525                 vtable.rank = 1;
5526
5527                 array_fill_vtable = &vtable;
5528         }
5529         return array_fill_vtable;
5530 }
5531
5532 void
5533 sgen_gc_lock (void)
5534 {
5535         LOCK_GC;
5536 }
5537
5538 void
5539 sgen_gc_unlock (void)
5540 {
5541         UNLOCK_GC;
5542 }
5543
5544 void
5545 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5546 {
5547         major_collector.iterate_live_block_ranges (callback);
5548 }
5549
5550 void
5551 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5552 {
5553         major_collector.scan_card_table (FALSE, queue);
5554 }
5555
5556 SgenMajorCollector*
5557 sgen_get_major_collector (void)
5558 {
5559         return &major_collector;
5560 }
5561
5562 void mono_gc_set_skip_thread (gboolean skip)
5563 {
5564         SgenThreadInfo *info = mono_thread_info_current ();
5565
5566         LOCK_GC;
5567         info->gc_disabled = skip;
5568         UNLOCK_GC;
5569 }
5570
5571 SgenRemeberedSet*
5572 sgen_get_remset (void)
5573 {
5574         return &remset;
5575 }
5576
5577 guint
5578 mono_gc_get_vtable_bits (MonoClass *class)
5579 {
5580         if (sgen_need_bridge_processing () && sgen_is_bridge_class (class))
5581                 return SGEN_GC_BIT_BRIDGE_OBJECT;
5582         return 0;
5583 }
5584
5585 void
5586 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5587 {
5588         // FIXME:
5589 }
5590
5591
5592 void
5593 sgen_check_whole_heap_stw (void)
5594 {
5595         sgen_stop_world (0);
5596         sgen_clear_nursery_fragments ();
5597         sgen_check_whole_heap (FALSE);
5598         sgen_restart_world (0, NULL);
5599 }
5600
5601 void
5602 sgen_gc_event_moves (void)
5603 {
5604         if (moved_objects_idx) {
5605                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5606                 moved_objects_idx = 0;
5607         }
5608 }
5609
5610 #endif /* HAVE_SGEN_GC */