mono/metadata/sgen-gc.c

   1 /*
   2  * sgen-gc.c: Simple generational GC.
   3  *
   4  * Author:
   5  *      Paolo Molaro (lupus@ximian.com)
   6  *  Rodrigo Kumpera (kumpera@gmail.com)
   7  *
   8  * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
   9  * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
  10  *
  11  * Thread start/stop adapted from Boehm's GC:
  12  * Copyright (c) 1994 by Xerox Corporation.  All rights reserved.
  13  * Copyright (c) 1996 by Silicon Graphics.  All rights reserved.
  14  * Copyright (c) 1998 by Fergus Henderson.  All rights reserved.
  15  * Copyright (c) 2000-2004 by Hewlett-Packard Company.  All rights reserved.
  16  * Copyright 2001-2003 Ximian, Inc
  17  * Copyright 2003-2010 Novell, Inc.
  18  * Copyright 2011 Xamarin, Inc.
  19  * Copyright (C) 2012 Xamarin Inc
  20  *
  21  * This library is free software; you can redistribute it and/or
  22  * modify it under the terms of the GNU Library General Public
  23  * License 2.0 as published by the Free Software Foundation;
  24  *
  25  * This library is distributed in the hope that it will be useful,
  26  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  28  * Library General Public License for more details.
  29  *
  30  * You should have received a copy of the GNU Library General Public
  31  * License 2.0 along with this library; if not, write to the Free
  32  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  33  *
  34  * Important: allocation provides always zeroed memory, having to do
  35  * a memset after allocation is deadly for performance.
  36  * Memory usage at startup is currently as follows:
  37  * 64 KB pinned space
  38  * 64 KB internal space
  39  * size of nursery
  40  * We should provide a small memory config with half the sizes
  41  *
  42  * We currently try to make as few mono assumptions as possible:
  43  * 1) 2-word header with no GC pointers in it (first vtable, second to store the
  44  *    forwarding ptr)
  45  * 2) gc descriptor is the second word in the vtable (first word in the class)
  46  * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
  47  * 4) there is a function to get an object's size and the number of
  48  *    elements in an array.
  49  * 5) we know the special way bounds are allocated for complex arrays
  50  * 6) we know about proxies and how to treat them when domains are unloaded
  51  *
  52  * Always try to keep stack usage to a minimum: no recursive behaviour
  53  * and no large stack allocs.
  54  *
  55  * General description.
  56  * Objects are initially allocated in a nursery using a fast bump-pointer technique.
  57  * When the nursery is full we start a nursery collection: this is performed with a
  58  * copying GC.
  59  * When the old generation is full we start a copying GC of the old generation as well:
  60  * this will be changed to mark&sweep with copying when fragmentation becomes to severe
  61  * in the future.  Maybe we'll even do both during the same collection like IMMIX.
  62  *
  63  * The things that complicate this description are:
  64  * *) pinned objects: we can't move them so we need to keep track of them
  65  * *) no precise info of the thread stacks and registers: we need to be able to
  66  *    quickly find the objects that may be referenced conservatively and pin them
  67  *    (this makes the first issues more important)
  68  * *) large objects are too expensive to be dealt with using copying GC: we handle them
  69  *    with mark/sweep during major collections
  70  * *) some objects need to not move even if they are small (interned strings, Type handles):
  71  *    we use mark/sweep for them, too: they are not allocated in the nursery, but inside
  72  *    PinnedChunks regions
  73  */
  74
  75 /*
  76  * TODO:
  77
  78  *) we could have a function pointer in MonoClass to implement
  79   customized write barriers for value types
  80
  81  *) investigate the stuff needed to advance a thread to a GC-safe
  82   point (single-stepping, read from unmapped memory etc) and implement it.
  83   This would enable us to inline allocations and write barriers, for example,
  84   or at least parts of them, like the write barrier checks.
  85   We may need this also for handling precise info on stacks, even simple things
  86   as having uninitialized data on the stack and having to wait for the prolog
  87   to zero it. Not an issue for the last frame that we scan conservatively.
  88   We could always not trust the value in the slots anyway.
  89
  90  *) modify the jit to save info about references in stack locations:
  91   this can be done just for locals as a start, so that at least
  92   part of the stack is handled precisely.
  93
  94  *) test/fix endianess issues
  95
  96  *) Implement a card table as the write barrier instead of remembered
  97     sets?  Card tables are not easy to implement with our current
  98     memory layout.  We have several different kinds of major heap
  99     objects: Small objects in regular blocks, small objects in pinned
 100     chunks and LOS objects.  If we just have a pointer we have no way
 101     to tell which kind of object it points into, therefore we cannot
 102     know where its card table is.  The least we have to do to make
 103     this happen is to get rid of write barriers for indirect stores.
 104     (See next item)
 105
 106  *) Get rid of write barriers for indirect stores.  We can do this by
 107     telling the GC to wbarrier-register an object once we do an ldloca
 108     or ldelema on it, and to unregister it once it's not used anymore
 109     (it can only travel downwards on the stack).  The problem with
 110     unregistering is that it needs to happen eventually no matter
 111     what, even if exceptions are thrown, the thread aborts, etc.
 112     Rodrigo suggested that we could do only the registering part and
 113     let the collector find out (pessimistically) when it's safe to
 114     unregister, namely when the stack pointer of the thread that
 115     registered the object is higher than it was when the registering
 116     happened.  This might make for a good first implementation to get
 117     some data on performance.
 118
 119  *) Some sort of blacklist support?  Blacklists is a concept from the
 120     Boehm GC: if during a conservative scan we find pointers to an
 121     area which we might use as heap, we mark that area as unusable, so
 122     pointer retention by random pinning pointers is reduced.
 123
 124  *) experiment with max small object size (very small right now - 2kb,
 125     because it's tied to the max freelist size)
 126
 127   *) add an option to mmap the whole heap in one chunk: it makes for many
 128      simplifications in the checks (put the nursery at the top and just use a single
 129      check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
 130      not flexible (too much of the address space may be used by default or we can't
 131      increase the heap as needed) and we'd need a race-free mechanism to return memory
 132      back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
 133      was written to, munmap is needed, but the following mmap may not find the same segment
 134      free...)
 135
 136  *) memzero the major fragments after restarting the world and optionally a smaller
 137     chunk at a time
 138
 139  *) investigate having fragment zeroing threads
 140
 141  *) separate locks for finalization and other minor stuff to reduce
 142     lock contention
 143
 144  *) try a different copying order to improve memory locality
 145
 146  *) a thread abort after a store but before the write barrier will
 147     prevent the write barrier from executing
 148
 149  *) specialized dynamically generated markers/copiers
 150
 151  *) Dynamically adjust TLAB size to the number of threads.  If we have
 152     too many threads that do allocation, we might need smaller TLABs,
 153     and we might get better performance with larger TLABs if we only
 154     have a handful of threads.  We could sum up the space left in all
 155     assigned TLABs and if that's more than some percentage of the
 156     nursery size, reduce the TLAB size.
 157
 158  *) Explore placing unreachable objects on unused nursery memory.
 159         Instead of memset'ng a region to zero, place an int[] covering it.
 160         A good place to start is add_nursery_frag. The tricky thing here is
 161         placing those objects atomically outside of a collection.
 162
 163  *) Allocation should use asymmetric Dekker synchronization:
 164         http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
 165         This should help weak consistency archs.
 166  */
 167 #include "config.h"
 168 #ifdef HAVE_SGEN_GC
 169
 170 #ifdef __MACH__
 171 #undef _XOPEN_SOURCE
 172 #define _XOPEN_SOURCE
 173 #define _DARWIN_C_SOURCE
 174 #endif
 175
 176 #ifdef HAVE_UNISTD_H
 177 #include <unistd.h>
 178 #endif
 179 #ifdef HAVE_PTHREAD_H
 180 #include <pthread.h>
 181 #endif
 182 #ifdef HAVE_PTHREAD_NP_H
 183 #include <pthread_np.h>
 184 #endif
 185 #ifdef HAVE_SEMAPHORE_H
 186 #include <semaphore.h>
 187 #endif
 188 #include <stdio.h>
 189 #include <string.h>
 190 #include <signal.h>
 191 #include <errno.h>
 192 #include <assert.h>
 193
 194 #include "metadata/sgen-gc.h"
 195 #include "metadata/metadata-internals.h"
 196 #include "metadata/class-internals.h"
 197 #include "metadata/gc-internal.h"
 198 #include "metadata/object-internals.h"
 199 #include "metadata/threads.h"
 200 #include "metadata/sgen-cardtable.h"
 201 #include "metadata/sgen-protocol.h"
 202 #include "metadata/sgen-archdep.h"
 203 #include "metadata/sgen-bridge.h"
 204 #include "metadata/sgen-memory-governor.h"
 205 #include "metadata/sgen-hash-table.h"
 206 #include "metadata/mono-gc.h"
 207 #include "metadata/method-builder.h"
 208 #include "metadata/profiler-private.h"
 209 #include "metadata/monitor.h"
 210 #include "metadata/threadpool-internals.h"
 211 #include "metadata/mempool-internals.h"
 212 #include "metadata/marshal.h"
 213 #include "metadata/runtime.h"
 214 #include "metadata/sgen-cardtable.h"
 215 #include "metadata/sgen-pinning.h"
 216 #include "metadata/sgen-workers.h"
 217 #include "metadata/sgen-layout-stats.h"
 218 #include "utils/mono-mmap.h"
 219 #include "utils/mono-time.h"
 220 #include "utils/mono-semaphore.h"
 221 #include "utils/mono-counters.h"
 222 #include "utils/mono-proclib.h"
 223 #include "utils/mono-memory-model.h"
 224 #include "utils/mono-logger-internal.h"
 225 #include "utils/dtrace.h"
 226
 227 #include <mono/utils/mono-logger-internal.h>
 228 #include <mono/utils/memcheck.h>
 229
 230 #if defined(__MACH__)
 231 #include "utils/mach-support.h"
 232 #endif
 233
 234 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
 235         a = i,
 236
 237 enum {
 238 #include "mono/cil/opcode.def"
 239         CEE_LAST
 240 };
 241
 242 #undef OPDEF
 243
 244 #undef pthread_create
 245 #undef pthread_join
 246 #undef pthread_detach
 247
 248 /*
 249  * ######################################################################
 250  * ########  Types and constants used by the GC.
 251  * ######################################################################
 252  */
 253
 254 /* 0 means not initialized, 1 is initialized, -1 means in progress */
 255 static int gc_initialized = 0;
 256 /* If set, check if we need to do something every X allocations */
 257 gboolean has_per_allocation_action;
 258 /* If set, do a heap check every X allocation */
 259 guint32 verify_before_allocs = 0;
 260 /* If set, do a minor collection before every X allocation */
 261 guint32 collect_before_allocs = 0;
 262 /* If set, do a whole heap check before each collection */
 263 static gboolean whole_heap_check_before_collection = FALSE;
 264 /* If set, do a heap consistency check before each minor collection */
 265 static gboolean consistency_check_at_minor_collection = FALSE;
 266 /* If set, do a mod union consistency check before each finishing collection pause */
 267 static gboolean mod_union_consistency_check = FALSE;
 268 /* If set, check whether mark bits are consistent after major collections */
 269 static gboolean check_mark_bits_after_major_collection = FALSE;
 270 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
 271 static gboolean check_nursery_objects_pinned = FALSE;
 272 /* If set, do a few checks when the concurrent collector is used */
 273 static gboolean do_concurrent_checks = FALSE;
 274 /* If set, check that there are no references to the domain left at domain unload */
 275 static gboolean xdomain_checks = FALSE;
 276 /* If not null, dump the heap after each collection into this file */
 277 static FILE *heap_dump_file = NULL;
 278 /* If set, mark stacks conservatively, even if precise marking is possible */
 279 static gboolean conservative_stack_mark = FALSE;
 280 /* If set, do a plausibility check on the scan_starts before and after
 281    each collection */
 282 static gboolean do_scan_starts_check = FALSE;
 283 /*
 284  * If the major collector is concurrent and this is FALSE, we will
 285  * never initiate a synchronous major collection, unless requested via
 286  * GC.Collect().
 287  */
 288 static gboolean allow_synchronous_major = TRUE;
 289 static gboolean nursery_collection_is_parallel = FALSE;
 290 static gboolean disable_minor_collections = FALSE;
 291 static gboolean disable_major_collections = FALSE;
 292 gboolean do_pin_stats = FALSE;
 293 static gboolean do_verify_nursery = FALSE;
 294 static gboolean do_dump_nursery_content = FALSE;
 295
 296 #ifdef HEAVY_STATISTICS
 297 long long stat_objects_alloced_degraded = 0;
 298 long long stat_bytes_alloced_degraded = 0;
 299
 300 long long stat_copy_object_called_nursery = 0;
 301 long long stat_objects_copied_nursery = 0;
 302 long long stat_copy_object_called_major = 0;
 303 long long stat_objects_copied_major = 0;
 304
 305 long long stat_scan_object_called_nursery = 0;
 306 long long stat_scan_object_called_major = 0;
 307
 308 long long stat_slots_allocated_in_vain;
 309
 310 long long stat_nursery_copy_object_failed_from_space = 0;
 311 long long stat_nursery_copy_object_failed_forwarded = 0;
 312 long long stat_nursery_copy_object_failed_pinned = 0;
 313 long long stat_nursery_copy_object_failed_to_space = 0;
 314
 315 static int stat_wbarrier_add_to_global_remset = 0;
 316 static int stat_wbarrier_set_field = 0;
 317 static int stat_wbarrier_set_arrayref = 0;
 318 static int stat_wbarrier_arrayref_copy = 0;
 319 static int stat_wbarrier_generic_store = 0;
 320 static int stat_wbarrier_set_root = 0;
 321 static int stat_wbarrier_value_copy = 0;
 322 static int stat_wbarrier_object_copy = 0;
 323 #endif
 324
 325 int stat_minor_gcs = 0;
 326 int stat_major_gcs = 0;
 327
 328 static long long stat_pinned_objects = 0;
 329
 330 static long long time_minor_pre_collection_fragment_clear = 0;
 331 static long long time_minor_pinning = 0;
 332 static long long time_minor_scan_remsets = 0;
 333 static long long time_minor_scan_pinned = 0;
 334 static long long time_minor_scan_registered_roots = 0;
 335 static long long time_minor_scan_thread_data = 0;
 336 static long long time_minor_finish_gray_stack = 0;
 337 static long long time_minor_fragment_creation = 0;
 338
 339 static long long time_major_pre_collection_fragment_clear = 0;
 340 static long long time_major_pinning = 0;
 341 static long long time_major_scan_pinned = 0;
 342 static long long time_major_scan_registered_roots = 0;
 343 static long long time_major_scan_thread_data = 0;
 344 static long long time_major_scan_alloc_pinned = 0;
 345 static long long time_major_scan_finalized = 0;
 346 static long long time_major_scan_big_objects = 0;
 347 static long long time_major_finish_gray_stack = 0;
 348 static long long time_major_free_bigobjs = 0;
 349 static long long time_major_los_sweep = 0;
 350 static long long time_major_sweep = 0;
 351 static long long time_major_fragment_creation = 0;
 352
 353 int gc_debug_level = 0;
 354 FILE* gc_debug_file;
 355
 356 /*
 357 void
 358 mono_gc_flush_info (void)
 359 {
 360         fflush (gc_debug_file);
 361 }
 362 */
 363
 364 #define TV_DECLARE SGEN_TV_DECLARE
 365 #define TV_GETTIME SGEN_TV_GETTIME
 366 #define TV_ELAPSED SGEN_TV_ELAPSED
 367 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
 368
 369 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
 370
 371 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
 372
 373 #define object_is_forwarded     SGEN_OBJECT_IS_FORWARDED
 374 #define object_is_pinned        SGEN_OBJECT_IS_PINNED
 375 #define pin_object              SGEN_PIN_OBJECT
 376 #define unpin_object            SGEN_UNPIN_OBJECT
 377
 378 #define ptr_in_nursery sgen_ptr_in_nursery
 379
 380 #define LOAD_VTABLE     SGEN_LOAD_VTABLE
 381
 382 static const char*
 383 safe_name (void* obj)
 384 {
 385         MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
 386         return vt->klass->name;
 387 }
 388
 389 #define safe_object_get_size    sgen_safe_object_get_size
 390
 391 const char*
 392 sgen_safe_name (void* obj)
 393 {
 394         return safe_name (obj);
 395 }
 396
 397 /*
 398  * ######################################################################
 399  * ########  Global data.
 400  * ######################################################################
 401  */
 402 LOCK_DECLARE (gc_mutex);
 403
 404 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
 405
 406 static mword pagesize = 4096;
 407 int degraded_mode = 0;
 408
 409 static mword bytes_pinned_from_failed_allocation = 0;
 410
 411 GCMemSection *nursery_section = NULL;
 412 static mword lowest_heap_address = ~(mword)0;
 413 static mword highest_heap_address = 0;
 414
 415 LOCK_DECLARE (sgen_interruption_mutex);
 416 static LOCK_DECLARE (pin_queue_mutex);
 417
 418 #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex)
 419 #define UNLOCK_PIN_QUEUE mono_mutex_unlock (&pin_queue_mutex)
 420
 421 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
 422 struct _FinalizeReadyEntry {
 423         FinalizeReadyEntry *next;
 424         void *object;
 425 };
 426
 427 typedef struct _EphemeronLinkNode EphemeronLinkNode;
 428
 429 struct _EphemeronLinkNode {
 430         EphemeronLinkNode *next;
 431         char *array;
 432 };
 433
 434 typedef struct {
 435        void *key;
 436        void *value;
 437 } Ephemeron;
 438
 439 int current_collection_generation = -1;
 440 volatile gboolean concurrent_collection_in_progress = FALSE;
 441
 442 /* objects that are ready to be finalized */
 443 static FinalizeReadyEntry *fin_ready_list = NULL;
 444 static FinalizeReadyEntry *critical_fin_list = NULL;
 445
 446 static EphemeronLinkNode *ephemeron_list;
 447
 448 /* registered roots: the key to the hash is the root start address */
 449 /*
 450  * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
 451  */
 452 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
 453         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 454         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
 455         SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
 456 };
 457 static mword roots_size = 0; /* amount of memory in the root set */
 458
 459 #define GC_ROOT_NUM 32
 460 typedef struct {
 461         int count;              /* must be the first field */
 462         void *objects [GC_ROOT_NUM];
 463         int root_types [GC_ROOT_NUM];
 464         uintptr_t extra_info [GC_ROOT_NUM];
 465 } GCRootReport;
 466
 467 static void
 468 notify_gc_roots (GCRootReport *report)
 469 {
 470         if (!report->count)
 471                 return;
 472         mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
 473         report->count = 0;
 474 }
 475
 476 static void
 477 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
 478 {
 479         if (report->count == GC_ROOT_NUM)
 480                 notify_gc_roots (report);
 481         report->objects [report->count] = object;
 482         report->root_types [report->count] = rtype;
 483         report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
 484 }
 485
 486 MonoNativeTlsKey thread_info_key;
 487
 488 #ifdef HAVE_KW_THREAD
 489 __thread SgenThreadInfo *sgen_thread_info;
 490 __thread char *stack_end;
 491 #endif
 492
 493 /* The size of a TLAB */
 494 /* The bigger the value, the less often we have to go to the slow path to allocate a new
 495  * one, but the more space is wasted by threads not allocating much memory.
 496  * FIXME: Tune this.
 497  * FIXME: Make this self-tuning for each thread.
 498  */
 499 guint32 tlab_size = (1024 * 4);
 500
 501 #define MAX_SMALL_OBJ_SIZE      SGEN_MAX_SMALL_OBJ_SIZE
 502
 503 /* Functions supplied by the runtime to be called by the GC */
 504 static MonoGCCallbacks gc_callbacks;
 505
 506 #define ALLOC_ALIGN             SGEN_ALLOC_ALIGN
 507 #define ALLOC_ALIGN_BITS        SGEN_ALLOC_ALIGN_BITS
 508
 509 #define ALIGN_UP                SGEN_ALIGN_UP
 510
 511 #define MOVED_OBJECTS_NUM 64
 512 static void *moved_objects [MOVED_OBJECTS_NUM];
 513 static int moved_objects_idx = 0;
 514
 515 /* Vtable of the objects used to fill out nursery fragments before a collection */
 516 static MonoVTable *array_fill_vtable;
 517
 518 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
 519 MonoNativeThreadId main_gc_thread = NULL;
 520 #endif
 521
 522 /*Object was pinned during the current collection*/
 523 static mword objects_pinned;
 524
 525 /*
 526  * ######################################################################
 527  * ########  Macros and function declarations.
 528  * ######################################################################
 529  */
 530
 531 inline static void*
 532 align_pointer (void *ptr)
 533 {
 534         mword p = (mword)ptr;
 535         p += sizeof (gpointer) - 1;
 536         p &= ~ (sizeof (gpointer) - 1);
 537         return (void*)p;
 538 }
 539
 540 typedef SgenGrayQueue GrayQueue;
 541
 542 /* forward declarations */
 543 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
 544 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
 545 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
 546 static void report_finalizer_roots (void);
 547 static void report_registered_roots (void);
 548
 549 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
 550 static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx);
 551 static void finish_gray_stack (int generation, GrayQueue *queue);
 552
 553 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
 554
 555
 556 static void init_stats (void);
 557
 558 static int mark_ephemerons_in_range (ScanCopyContext ctx);
 559 static void clear_unreachable_ephemerons (ScanCopyContext ctx);
 560 static void null_ephemerons_for_domain (MonoDomain *domain);
 561
 562 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
 563
 564 SgenObjectOperations current_object_ops;
 565 SgenMajorCollector major_collector;
 566 SgenMinorCollector sgen_minor_collector;
 567 static GrayQueue gray_queue;
 568
 569 static SgenRemeberedSet remset;
 570
 571 /* The gray queue to use from the main collection thread. */
 572 #define WORKERS_DISTRIBUTE_GRAY_QUEUE   (&gray_queue)
 573
 574 /*
 575  * The gray queue a worker job must use.  If we're not parallel or
 576  * concurrent, we use the main gray queue.
 577  */
 578 static SgenGrayQueue*
 579 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
 580 {
 581         return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
 582 }
 583
 584 static void
 585 gray_queue_redirect (SgenGrayQueue *queue)
 586 {
 587         gboolean wake = FALSE;
 588
 589
 590         for (;;) {
 591                 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
 592                 if (!section)
 593                         break;
 594                 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
 595                 wake = TRUE;
 596         }
 597
 598         if (wake) {
 599                 g_assert (concurrent_collection_in_progress ||
 600                                 (current_collection_generation == GENERATION_OLD && major_collector.is_parallel));
 601                 if (sgen_workers_have_started ()) {
 602                         sgen_workers_wake_up_all ();
 603                 } else {
 604                         if (concurrent_collection_in_progress)
 605                                 g_assert (current_collection_generation == -1);
 606                 }
 607         }
 608 }
 609
 610 static gboolean
 611 is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
 612 {
 613         MonoObject *o = (MonoObject*)(obj);
 614         MonoObject *ref = (MonoObject*)*(ptr);
 615         int offset = (char*)(ptr) - (char*)o;
 616
 617         if (o->vtable->klass == mono_defaults.thread_class && offset == G_STRUCT_OFFSET (MonoThread, internal_thread))
 618                 return TRUE;
 619         if (o->vtable->klass == mono_defaults.internal_thread_class && offset == G_STRUCT_OFFSET (MonoInternalThread, current_appcontext))
 620                 return TRUE;
 621
 622 #ifndef DISABLE_REMOTING
 623         if (mono_class_has_parent_fast (o->vtable->klass, mono_defaults.real_proxy_class) &&
 624                         offset == G_STRUCT_OFFSET (MonoRealProxy, unwrapped_server))
 625                 return TRUE;
 626 #endif
 627         /* Thread.cached_culture_info */
 628         if (!strcmp (ref->vtable->klass->name_space, "System.Globalization") &&
 629                         !strcmp (ref->vtable->klass->name, "CultureInfo") &&
 630                         !strcmp(o->vtable->klass->name_space, "System") &&
 631                         !strcmp(o->vtable->klass->name, "Object[]"))
 632                 return TRUE;
 633         /*
 634          *  at System.IO.MemoryStream.InternalConstructor (byte[],int,int,bool,bool) [0x0004d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:121
 635          * at System.IO.MemoryStream..ctor (byte[]) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.IO/MemoryStream.cs:81
 636          * at (wrapper remoting-invoke-with-check) System.IO.MemoryStream..ctor (byte[]) <IL 0x00020, 0xffffffff>
 637          * at System.Runtime.Remoting.Messaging.CADMethodCallMessage.GetArguments () [0x0000d] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/CADMessages.cs:327
 638          * at System.Runtime.Remoting.Messaging.MethodCall..ctor (System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00017] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Messaging/MethodCall.cs:87
 639          * at System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) [0x00018] in /home/schani/Work/novell/trunk/mcs/class/corlib/System/AppDomain.cs:1213
 640          * at (wrapper remoting-invoke-with-check) System.AppDomain.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage,byte[]&,System.Runtime.Remoting.Messaging.CADMethodReturnMessage&) <IL 0x0003d, 0xffffffff>
 641          * at System.Runtime.Remoting.Channels.CrossAppDomainSink.ProcessMessageInDomain (byte[],System.Runtime.Remoting.Messaging.CADMethodCallMessage) [0x00008] in /home/schani/Work/novell/trunk/mcs/class/corlib/System.Runtime.Remoting.Channels/CrossAppDomainChannel.cs:198
 642          * at (wrapper runtime-invoke) object.runtime_invoke_CrossAppDomainSink/ProcessMessageRes_object_object (object,intptr,intptr,intptr) <IL 0x0004c, 0xffffffff>
 643          */
 644         if (!strcmp (ref->vtable->klass->name_space, "System") &&
 645                         !strcmp (ref->vtable->klass->name, "Byte[]") &&
 646                         !strcmp (o->vtable->klass->name_space, "System.IO") &&
 647                         !strcmp (o->vtable->klass->name, "MemoryStream"))
 648                 return TRUE;
 649         /* append_job() in threadpool.c */
 650         if (!strcmp (ref->vtable->klass->name_space, "System.Runtime.Remoting.Messaging") &&
 651                         !strcmp (ref->vtable->klass->name, "AsyncResult") &&
 652                         !strcmp (o->vtable->klass->name_space, "System") &&
 653                         !strcmp (o->vtable->klass->name, "Object[]") &&
 654                         mono_thread_pool_is_queue_array ((MonoArray*) o))
 655                 return TRUE;
 656         return FALSE;
 657 }
 658
 659 static void
 660 check_reference_for_xdomain (gpointer *ptr, char *obj, MonoDomain *domain)
 661 {
 662         MonoObject *o = (MonoObject*)(obj);
 663         MonoObject *ref = (MonoObject*)*(ptr);
 664         int offset = (char*)(ptr) - (char*)o;
 665         MonoClass *class;
 666         MonoClassField *field;
 667         char *str;
 668
 669         if (!ref || ref->vtable->domain == domain)
 670                 return;
 671         if (is_xdomain_ref_allowed (ptr, obj, domain))
 672                 return;
 673
 674         field = NULL;
 675         for (class = o->vtable->klass; class; class = class->parent) {
 676                 int i;
 677
 678                 for (i = 0; i < class->field.count; ++i) {
 679                         if (class->fields[i].offset == offset) {
 680                                 field = &class->fields[i];
 681                                 break;
 682                         }
 683                 }
 684                 if (field)
 685                         break;
 686         }
 687
 688         if (ref->vtable->klass == mono_defaults.string_class)
 689                 str = mono_string_to_utf8 ((MonoString*)ref);
 690         else
 691                 str = NULL;
 692         g_print ("xdomain reference in %p (%s.%s) at offset %d (%s) to %p (%s.%s) (%s)  -  pointed to by:\n",
 693                         o, o->vtable->klass->name_space, o->vtable->klass->name,
 694                         offset, field ? field->name : "",
 695                         ref, ref->vtable->klass->name_space, ref->vtable->klass->name, str ? str : "");
 696         mono_gc_scan_for_specific_ref (o, TRUE);
 697         if (str)
 698                 g_free (str);
 699 }
 700
 701 #undef HANDLE_PTR
 702 #define HANDLE_PTR(ptr,obj)     check_reference_for_xdomain ((ptr), (obj), domain)
 703
 704 static void
 705 scan_object_for_xdomain_refs (char *start, mword size, void *data)
 706 {
 707         MonoDomain *domain = ((MonoObject*)start)->vtable->domain;
 708
 709         #include "sgen-scan-object.h"
 710 }
 711
 712 static gboolean scan_object_for_specific_ref_precise = TRUE;
 713
 714 #undef HANDLE_PTR
 715 #define HANDLE_PTR(ptr,obj) do {                \
 716         if ((MonoObject*)*(ptr) == key) {       \
 717         g_print ("found ref to %p in object %p (%s) at offset %td\n",   \
 718                         key, (obj), safe_name ((obj)), ((char*)(ptr) - (char*)(obj))); \
 719         }                                                               \
 720         } while (0)
 721
 722 static void
 723 scan_object_for_specific_ref (char *start, MonoObject *key)
 724 {
 725         char *forwarded;
 726
 727         if ((forwarded = SGEN_OBJECT_IS_FORWARDED (start)))
 728                 start = forwarded;
 729
 730         if (scan_object_for_specific_ref_precise) {
 731                 #include "sgen-scan-object.h"
 732         } else {
 733                 mword *words = (mword*)start;
 734                 size_t size = safe_object_get_size ((MonoObject*)start);
 735                 int i;
 736                 for (i = 0; i < size / sizeof (mword); ++i) {
 737                         if (words [i] == (mword)key) {
 738                                 g_print ("found possible ref to %p in object %p (%s) at offset %td\n",
 739                                                 key, start, safe_name (start), i * sizeof (mword));
 740                         }
 741                 }
 742         }
 743 }
 744
 745 void
 746 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
 747 {
 748         while (start < end) {
 749                 size_t size;
 750                 char *obj;
 751
 752                 if (!*(void**)start) {
 753                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
 754                         continue;
 755                 }
 756
 757                 if (allow_flags) {
 758                         if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
 759                                 obj = start;
 760                 } else {
 761                         obj = start;
 762                 }
 763
 764                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
 765
 766                 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable)
 767                         callback (obj, size, data);
 768
 769                 start += size;
 770         }
 771 }
 772
 773 static void
 774 scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
 775 {
 776         scan_object_for_specific_ref (obj, key);
 777 }
 778
 779 static void
 780 check_root_obj_specific_ref (RootRecord *root, MonoObject *key, MonoObject *obj)
 781 {
 782         if (key != obj)
 783                 return;
 784         g_print ("found ref to %p in root record %p\n", key, root);
 785 }
 786
 787 static MonoObject *check_key = NULL;
 788 static RootRecord *check_root = NULL;
 789
 790 static void
 791 check_root_obj_specific_ref_from_marker (void **obj)
 792 {
 793         check_root_obj_specific_ref (check_root, check_key, *obj);
 794 }
 795
 796 static void
 797 scan_roots_for_specific_ref (MonoObject *key, int root_type)
 798 {
 799         void **start_root;
 800         RootRecord *root;
 801         check_key = key;
 802
 803         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 804                 mword desc = root->root_desc;
 805
 806                 check_root = root;
 807
 808                 switch (desc & ROOT_DESC_TYPE_MASK) {
 809                 case ROOT_DESC_BITMAP:
 810                         desc >>= ROOT_DESC_TYPE_SHIFT;
 811                         while (desc) {
 812                                 if (desc & 1)
 813                                         check_root_obj_specific_ref (root, key, *start_root);
 814                                 desc >>= 1;
 815                                 start_root++;
 816                         }
 817                         return;
 818                 case ROOT_DESC_COMPLEX: {
 819                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 820                         int bwords = (*bitmap_data) - 1;
 821                         void **start_run = start_root;
 822                         bitmap_data++;
 823                         while (bwords-- > 0) {
 824                                 gsize bmap = *bitmap_data++;
 825                                 void **objptr = start_run;
 826                                 while (bmap) {
 827                                         if (bmap & 1)
 828                                                 check_root_obj_specific_ref (root, key, *objptr);
 829                                         bmap >>= 1;
 830                                         ++objptr;
 831                                 }
 832                                 start_run += GC_BITS_PER_WORD;
 833                         }
 834                         break;
 835                 }
 836                 case ROOT_DESC_USER: {
 837                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 838                         marker (start_root, check_root_obj_specific_ref_from_marker);
 839                         break;
 840                 }
 841                 case ROOT_DESC_RUN_LEN:
 842                         g_assert_not_reached ();
 843                 default:
 844                         g_assert_not_reached ();
 845                 }
 846         } SGEN_HASH_TABLE_FOREACH_END;
 847
 848         check_key = NULL;
 849         check_root = NULL;
 850 }
 851
 852 void
 853 mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise)
 854 {
 855         void **ptr;
 856         RootRecord *root;
 857
 858         scan_object_for_specific_ref_precise = precise;
 859
 860         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 861                         (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key, TRUE);
 862
 863         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 864
 865         sgen_los_iterate_objects ((IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
 866
 867         scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
 868         scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
 869
 870         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], ptr, root) {
 871                 while (ptr < (void**)root->end_root) {
 872                         check_root_obj_specific_ref (root, *ptr, key);
 873                         ++ptr;
 874                 }
 875         } SGEN_HASH_TABLE_FOREACH_END;
 876 }
 877
 878 static gboolean
 879 need_remove_object_for_domain (char *start, MonoDomain *domain)
 880 {
 881         if (mono_object_domain (start) == domain) {
 882                 SGEN_LOG (4, "Need to cleanup object %p", start);
 883                 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
 884                 return TRUE;
 885         }
 886         return FALSE;
 887 }
 888
 889 static void
 890 process_object_for_domain_clearing (char *start, MonoDomain *domain)
 891 {
 892         GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
 893         if (vt->klass == mono_defaults.internal_thread_class)
 894                 g_assert (mono_object_domain (start) == mono_get_root_domain ());
 895         /* The object could be a proxy for an object in the domain
 896            we're deleting. */
 897 #ifndef DISABLE_REMOTING
 898         if (mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
 899                 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
 900
 901                 /* The server could already have been zeroed out, so
 902                    we need to check for that, too. */
 903                 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
 904                         SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
 905                         ((MonoRealProxy*)start)->unwrapped_server = NULL;
 906                 }
 907         }
 908 #endif
 909 }
 910
 911 static MonoDomain *check_domain = NULL;
 912
 913 static void
 914 check_obj_not_in_domain (void **o)
 915 {
 916         g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
 917 }
 918
 919 static void
 920 scan_for_registered_roots_in_domain (MonoDomain *domain, int root_type)
 921 {
 922         void **start_root;
 923         RootRecord *root;
 924         check_domain = domain;
 925         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
 926                 mword desc = root->root_desc;
 927
 928                 /* The MonoDomain struct is allowed to hold
 929                    references to objects in its own domain. */
 930                 if (start_root == (void**)domain)
 931                         continue;
 932
 933                 switch (desc & ROOT_DESC_TYPE_MASK) {
 934                 case ROOT_DESC_BITMAP:
 935                         desc >>= ROOT_DESC_TYPE_SHIFT;
 936                         while (desc) {
 937                                 if ((desc & 1) && *start_root)
 938                                         check_obj_not_in_domain (*start_root);
 939                                 desc >>= 1;
 940                                 start_root++;
 941                         }
 942                         break;
 943                 case ROOT_DESC_COMPLEX: {
 944                         gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
 945                         int bwords = (*bitmap_data) - 1;
 946                         void **start_run = start_root;
 947                         bitmap_data++;
 948                         while (bwords-- > 0) {
 949                                 gsize bmap = *bitmap_data++;
 950                                 void **objptr = start_run;
 951                                 while (bmap) {
 952                                         if ((bmap & 1) && *objptr)
 953                                                 check_obj_not_in_domain (*objptr);
 954                                         bmap >>= 1;
 955                                         ++objptr;
 956                                 }
 957                                 start_run += GC_BITS_PER_WORD;
 958                         }
 959                         break;
 960                 }
 961                 case ROOT_DESC_USER: {
 962                         MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
 963                         marker (start_root, check_obj_not_in_domain);
 964                         break;
 965                 }
 966                 case ROOT_DESC_RUN_LEN:
 967                         g_assert_not_reached ();
 968                 default:
 969                         g_assert_not_reached ();
 970                 }
 971         } SGEN_HASH_TABLE_FOREACH_END;
 972
 973         check_domain = NULL;
 974 }
 975
 976 static void
 977 check_for_xdomain_refs (void)
 978 {
 979         LOSObject *bigobj;
 980
 981         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
 982                         (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL, FALSE);
 983
 984         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL);
 985
 986         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
 987                 scan_object_for_xdomain_refs (bigobj->data, sgen_los_object_size (bigobj), NULL);
 988 }
 989
 990 static gboolean
 991 clear_domain_process_object (char *obj, MonoDomain *domain)
 992 {
 993         gboolean remove;
 994
 995         process_object_for_domain_clearing (obj, domain);
 996         remove = need_remove_object_for_domain (obj, domain);
 997
 998         if (remove && ((MonoObject*)obj)->synchronisation) {
 999                 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
1000                 if (dislink)
1001                         sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
1002         }
1003
1004         return remove;
1005 }
1006
1007 static void
1008 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
1009 {
1010         if (clear_domain_process_object (obj, domain))
1011                 memset (obj, 0, size);
1012 }
1013
1014 static void
1015 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
1016 {
1017         clear_domain_process_object (obj, domain);
1018 }
1019
1020 static void
1021 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1022 {
1023         if (need_remove_object_for_domain (obj, domain))
1024                 major_collector.free_non_pinned_object (obj, size);
1025 }
1026
1027 static void
1028 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
1029 {
1030         if (need_remove_object_for_domain (obj, domain))
1031                 major_collector.free_pinned_object (obj, size);
1032 }
1033
1034 /*
1035  * When appdomains are unloaded we can easily remove objects that have finalizers,
1036  * but all the others could still be present in random places on the heap.
1037  * We need a sweep to get rid of them even though it's going to be costly
1038  * with big heaps.
1039  * The reason we need to remove them is because we access the vtable and class
1040  * structures to know the object size and the reference bitmap: once the domain is
1041  * unloaded the point to random memory.
1042  */
1043 void
1044 mono_gc_clear_domain (MonoDomain * domain)
1045 {
1046         LOSObject *bigobj, *prev;
1047         int i;
1048
1049         LOCK_GC;
1050
1051         binary_protocol_domain_unload_begin (domain);
1052
1053         sgen_stop_world (0);
1054
1055         if (concurrent_collection_in_progress)
1056                 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
1057         g_assert (!concurrent_collection_in_progress);
1058
1059         sgen_process_fin_stage_entries ();
1060         sgen_process_dislink_stage_entries ();
1061
1062         sgen_clear_nursery_fragments ();
1063
1064         if (xdomain_checks && domain != mono_get_root_domain ()) {
1065                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
1066                 scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
1067                 check_for_xdomain_refs ();
1068         }
1069
1070         /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
1071         to memory returned to the OS.*/
1072         null_ephemerons_for_domain (domain);
1073
1074         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1075                 sgen_null_links_for_domain (domain, i);
1076
1077         for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
1078                 sgen_remove_finalizers_for_domain (domain, i);
1079
1080         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
1081                         (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
1082
1083         /* We need two passes over major and large objects because
1084            freeing such objects might give their memory back to the OS
1085            (in the case of large objects) or obliterate its vtable
1086            (pinned objects with major-copying or pinned and non-pinned
1087            objects with major-mark&sweep), but we might need to
1088            dereference a pointer from an object to another object if
1089            the first object is a proxy. */
1090         major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
1091         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1092                 clear_domain_process_object (bigobj->data, domain);
1093
1094         prev = NULL;
1095         for (bigobj = los_object_list; bigobj;) {
1096                 if (need_remove_object_for_domain (bigobj->data, domain)) {
1097                         LOSObject *to_free = bigobj;
1098                         if (prev)
1099                                 prev->next = bigobj->next;
1100                         else
1101                                 los_object_list = bigobj->next;
1102                         bigobj = bigobj->next;
1103                         SGEN_LOG (4, "Freeing large object %p", bigobj->data);
1104                         sgen_los_free_object (to_free);
1105                         continue;
1106                 }
1107                 prev = bigobj;
1108                 bigobj = bigobj->next;
1109         }
1110         major_collector.iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
1111         major_collector.iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
1112
1113         if (domain == mono_get_root_domain ()) {
1114                 if (G_UNLIKELY (do_pin_stats))
1115                         sgen_pin_stats_print_class_stats ();
1116                 sgen_object_layout_dump (stdout);
1117         }
1118
1119         sgen_restart_world (0, NULL);
1120
1121         binary_protocol_domain_unload_end (domain);
1122
1123         UNLOCK_GC;
1124 }
1125
1126 /*
1127  * sgen_add_to_global_remset:
1128  *
1129  *   The global remset contains locations which point into newspace after
1130  * a minor collection. This can happen if the objects they point to are pinned.
1131  *
1132  * LOCKING: If called from a parallel collector, the global remset
1133  * lock must be held.  For serial collectors that is not necessary.
1134  */
1135 void
1136 sgen_add_to_global_remset (gpointer ptr, gpointer obj)
1137 {
1138         SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
1139
1140         HEAVY_STAT (++stat_wbarrier_add_to_global_remset);
1141
1142         if (!major_collector.is_concurrent) {
1143                 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
1144         } else {
1145                 if (current_collection_generation == -1)
1146                         SGEN_ASSERT (5, sgen_concurrent_collection_in_progress (), "Global remsets outside of collection pauses can only be added by the concurrent collector");
1147         }
1148
1149         if (!object_is_pinned (obj))
1150                 SGEN_ASSERT (5, sgen_minor_collector.is_split || sgen_concurrent_collection_in_progress (), "Non-pinned objects can only remain in nursery if it is a split nursery");
1151         else if (sgen_cement_lookup_or_register (obj))
1152                 return;
1153
1154         remset.record_pointer (ptr);
1155
1156         if (G_UNLIKELY (do_pin_stats))
1157                 sgen_pin_stats_register_global_remset (obj);
1158
1159         SGEN_LOG (8, "Adding global remset for %p", ptr);
1160         binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
1161
1162
1163 #ifdef ENABLE_DTRACE
1164         if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
1165                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
1166                 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
1167                                 vt->klass->name_space, vt->klass->name);
1168         }
1169 #endif
1170 }
1171
1172 /*
1173  * sgen_drain_gray_stack:
1174  *
1175  *   Scan objects in the gray stack until the stack is empty. This should be called
1176  * frequently after each object is copied, to achieve better locality and cache
1177  * usage.
1178  */
1179 gboolean
1180 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
1181 {
1182         char *obj;
1183         ScanObjectFunc scan_func = ctx.scan_func;
1184         GrayQueue *queue = ctx.queue;
1185
1186         if (max_objs == -1) {
1187                 for (;;) {
1188                         GRAY_OBJECT_DEQUEUE (queue, obj);
1189                         if (!obj)
1190                                 return TRUE;
1191                         SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1192                         scan_func (obj, queue);
1193                 }
1194         } else {
1195                 int i;
1196
1197                 do {
1198                         for (i = 0; i != max_objs; ++i) {
1199                                 GRAY_OBJECT_DEQUEUE (queue, obj);
1200                                 if (!obj)
1201                                         return TRUE;
1202                                 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
1203                                 scan_func (obj, queue);
1204                         }
1205                 } while (max_objs < 0);
1206                 return FALSE;
1207         }
1208 }
1209
1210 /*
1211  * Addresses from start to end are already sorted. This function finds
1212  * the object header for each address and pins the object. The
1213  * addresses must be inside the passed section.  The (start of the)
1214  * address array is overwritten with the addresses of the actually
1215  * pinned objects.  Return the number of pinned objects.
1216  */
1217 static int
1218 pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, ScanCopyContext ctx)
1219 {
1220         void *last = NULL;
1221         int count = 0;
1222         void *search_start;
1223         void *last_obj = NULL;
1224         size_t last_obj_size = 0;
1225         void *addr;
1226         int idx;
1227         void **definitely_pinned = start;
1228         ScanObjectFunc scan_func = ctx.scan_func;
1229         SgenGrayQueue *queue = ctx.queue;
1230
1231         sgen_nursery_allocator_prepare_for_pinning ();
1232
1233         while (start < end) {
1234                 addr = *start;
1235                 /* the range check should be reduntant */
1236                 if (addr != last && addr >= start_nursery && addr < end_nursery) {
1237                         SGEN_LOG (5, "Considering pinning addr %p", addr);
1238                         /* multiple pointers to the same object */
1239                         if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) {
1240                                 start++;
1241                                 continue;
1242                         }
1243                         idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
1244                         g_assert (idx < section->num_scan_start);
1245                         search_start = (void*)section->scan_starts [idx];
1246                         if (!search_start || search_start > addr) {
1247                                 while (idx) {
1248                                         --idx;
1249                                         search_start = section->scan_starts [idx];
1250                                         if (search_start && search_start <= addr)
1251                                                 break;
1252                                 }
1253                                 if (!search_start || search_start > addr)
1254                                         search_start = start_nursery;
1255                         }
1256                         if (search_start < last_obj)
1257                                 search_start = (char*)last_obj + last_obj_size;
1258                         /* now addr should be in an object a short distance from search_start
1259                          * Note that search_start must point to zeroed mem or point to an object.
1260                          */
1261
1262                         do {
1263                                 if (!*(void**)search_start) {
1264                                         /* Consistency check */
1265                                         /*
1266                                         for (frag = nursery_fragments; frag; frag = frag->next) {
1267                                                 if (search_start >= frag->fragment_start && search_start < frag->fragment_end)
1268                                                         g_assert_not_reached ();
1269                                         }
1270                                         */
1271
1272                                         search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
1273                                         continue;
1274                                 }
1275                                 last_obj = search_start;
1276                                 last_obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
1277
1278                                 if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) {
1279                                         /* Marks the beginning of a nursery fragment, skip */
1280                                 } else {
1281                                         SGEN_LOG (8, "Pinned try match %p (%s), size %zd", last_obj, safe_name (last_obj), last_obj_size);
1282                                         if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
1283                                                 if (scan_func) {
1284                                                         scan_func (search_start, queue);
1285                                                 } else {
1286                                                         SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1287                                                                         search_start, *(void**)search_start, safe_name (search_start), count);
1288                                                         binary_protocol_pin (search_start,
1289                                                                         (gpointer)LOAD_VTABLE (search_start),
1290                                                                         safe_object_get_size (search_start));
1291
1292 #ifdef ENABLE_DTRACE
1293                                                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1294                                                                 int gen = sgen_ptr_in_nursery (search_start) ? GENERATION_NURSERY : GENERATION_OLD;
1295                                                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (search_start);
1296                                                                 MONO_GC_OBJ_PINNED ((mword)search_start,
1297                                                                                 sgen_safe_object_get_size (search_start),
1298                                                                                 vt->klass->name_space, vt->klass->name, gen);
1299                                                         }
1300 #endif
1301
1302                                                         pin_object (search_start);
1303                                                         GRAY_OBJECT_ENQUEUE (queue, search_start);
1304                                                         if (G_UNLIKELY (do_pin_stats))
1305                                                                 sgen_pin_stats_register_object (search_start, last_obj_size);
1306                                                         definitely_pinned [count] = search_start;
1307                                                         count++;
1308                                                 }
1309                                                 break;
1310                                         }
1311                                 }
1312                                 /* skip to the next object */
1313                                 search_start = (void*)((char*)search_start + last_obj_size);
1314                         } while (search_start <= addr);
1315                         /* we either pinned the correct object or we ignored the addr because
1316                          * it points to unused zeroed memory.
1317                          */
1318                         last = addr;
1319                 }
1320                 start++;
1321         }
1322         //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1323         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1324                 GCRootReport report;
1325                 report.count = 0;
1326                 for (idx = 0; idx < count; ++idx)
1327                         add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1328                 notify_gc_roots (&report);
1329         }
1330         stat_pinned_objects += count;
1331         return count;
1332 }
1333
1334 void
1335 sgen_pin_objects_in_section (GCMemSection *section, ScanCopyContext ctx)
1336 {
1337         int num_entries = section->pin_queue_num_entries;
1338         if (num_entries) {
1339                 void **start = section->pin_queue_start;
1340                 int reduced_to;
1341                 reduced_to = pin_objects_from_addresses (section, start, start + num_entries,
1342                                 section->data, section->next_data, ctx);
1343                 section->pin_queue_num_entries = reduced_to;
1344                 if (!reduced_to)
1345                         section->pin_queue_start = NULL;
1346         }
1347 }
1348
1349
1350 void
1351 sgen_pin_object (void *object, GrayQueue *queue)
1352 {
1353         g_assert (!concurrent_collection_in_progress);
1354
1355         if (sgen_collection_is_parallel ()) {
1356                 LOCK_PIN_QUEUE;
1357                 /*object arrives pinned*/
1358                 sgen_pin_stage_ptr (object);
1359                 ++objects_pinned ;
1360                 UNLOCK_PIN_QUEUE;
1361         } else {
1362                 SGEN_PIN_OBJECT (object);
1363                 sgen_pin_stage_ptr (object);
1364                 ++objects_pinned;
1365                 if (G_UNLIKELY (do_pin_stats))
1366                         sgen_pin_stats_register_object (object, safe_object_get_size (object));
1367         }
1368         GRAY_OBJECT_ENQUEUE (queue, object);
1369         binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1370
1371 #ifdef ENABLE_DTRACE
1372         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1373                 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1374                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1375                 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1376         }
1377 #endif
1378 }
1379
1380 void
1381 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1382 {
1383         for (;;) {
1384                 mword vtable_word;
1385                 gboolean major_pinned = FALSE;
1386
1387                 if (sgen_ptr_in_nursery (obj)) {
1388                         if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
1389                                 sgen_pin_object (obj, queue);
1390                                 break;
1391                         }
1392                 } else {
1393                         major_collector.pin_major_object (obj, queue);
1394                         major_pinned = TRUE;
1395                 }
1396
1397                 vtable_word = *(mword*)obj;
1398                 /*someone else forwarded it, update the pointer and bail out*/
1399                 if (vtable_word & SGEN_FORWARDED_BIT) {
1400                         *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
1401                         break;
1402                 }
1403
1404                 /*someone pinned it, nothing to do.*/
1405                 if (vtable_word & SGEN_PINNED_BIT || major_pinned)
1406                         break;
1407         }
1408 }
1409
1410 /* Sort the addresses in array in increasing order.
1411  * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1412  */
1413 void
1414 sgen_sort_addresses (void **array, int size)
1415 {
1416         int i;
1417         void *tmp;
1418
1419         for (i = 1; i < size; ++i) {
1420                 int child = i;
1421                 while (child > 0) {
1422                         int parent = (child - 1) / 2;
1423
1424                         if (array [parent] >= array [child])
1425                                 break;
1426
1427                         tmp = array [parent];
1428                         array [parent] = array [child];
1429                         array [child] = tmp;
1430
1431                         child = parent;
1432                 }
1433         }
1434
1435         for (i = size - 1; i > 0; --i) {
1436                 int end, root;
1437                 tmp = array [i];
1438                 array [i] = array [0];
1439                 array [0] = tmp;
1440
1441                 end = i - 1;
1442                 root = 0;
1443
1444                 while (root * 2 + 1 <= end) {
1445                         int child = root * 2 + 1;
1446
1447                         if (child < end && array [child] < array [child + 1])
1448                                 ++child;
1449                         if (array [root] >= array [child])
1450                                 break;
1451
1452                         tmp = array [root];
1453                         array [root] = array [child];
1454                         array [child] = tmp;
1455
1456                         root = child;
1457                 }
1458         }
1459 }
1460
1461 /*
1462  * Scan the memory between start and end and queue values which could be pointers
1463  * to the area between start_nursery and end_nursery for later consideration.
1464  * Typically used for thread stacks.
1465  */
1466 static void
1467 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1468 {
1469         int count = 0;
1470
1471 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1472         VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1473 #endif
1474
1475         while (start < end) {
1476                 if (*start >= start_nursery && *start < end_nursery) {
1477                         /*
1478                          * *start can point to the middle of an object
1479                          * note: should we handle pointing at the end of an object?
1480                          * pinning in C# code disallows pointing at the end of an object
1481                          * but there is some small chance that an optimizing C compiler
1482                          * may keep the only reference to an object by pointing
1483                          * at the end of it. We ignore this small chance for now.
1484                          * Pointers to the end of an object are indistinguishable
1485                          * from pointers to the start of the next object in memory
1486                          * so if we allow that we'd need to pin two objects...
1487                          * We queue the pointer in an array, the
1488                          * array will then be sorted and uniqued. This way
1489                          * we can coalesce several pinning pointers and it should
1490                          * be faster since we'd do a memory scan with increasing
1491                          * addresses. Note: we can align the address to the allocation
1492                          * alignment, so the unique process is more effective.
1493                          */
1494                         mword addr = (mword)*start;
1495                         addr &= ~(ALLOC_ALIGN - 1);
1496                         if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1497                                 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1498                                 sgen_pin_stage_ptr ((void*)addr);
1499                                 count++;
1500                         }
1501                         if (G_UNLIKELY (do_pin_stats)) {
1502                                 if (ptr_in_nursery ((void*)addr))
1503                                         sgen_pin_stats_register_address ((char*)addr, pin_type);
1504                         }
1505                 }
1506                 start++;
1507         }
1508         if (count)
1509                 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1510 }
1511
1512 /*
1513  * The first thing we do in a collection is to identify pinned objects.
1514  * This function considers all the areas of memory that need to be
1515  * conservatively scanned.
1516  */
1517 static void
1518 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1519 {
1520         void **start_root;
1521         RootRecord *root;
1522         SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1523         /* objects pinned from the API are inside these roots */
1524         SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1525                 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1526                 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1527         } SGEN_HASH_TABLE_FOREACH_END;
1528         /* now deal with the thread stacks
1529          * in the future we should be able to conservatively scan only:
1530          * *) the cpu registers
1531          * *) the unmanaged stack frames
1532          * *) the _last_ managed stack frame
1533          * *) pointers slots in managed frames
1534          */
1535         scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1536 }
1537
1538 static void
1539 unpin_objects_from_queue (SgenGrayQueue *queue)
1540 {
1541         for (;;) {
1542                 char *addr;
1543                 GRAY_OBJECT_DEQUEUE (queue, addr);
1544                 if (!addr)
1545                         break;
1546                 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1547                 SGEN_UNPIN_OBJECT (addr);
1548         }
1549 }
1550
1551 typedef struct {
1552         CopyOrMarkObjectFunc func;
1553         GrayQueue *queue;
1554 } UserCopyOrMarkData;
1555
1556 static MonoNativeTlsKey user_copy_or_mark_key;
1557
1558 static void
1559 init_user_copy_or_mark_key (void)
1560 {
1561         mono_native_tls_alloc (&user_copy_or_mark_key, NULL);
1562 }
1563
1564 static void
1565 set_user_copy_or_mark_data (UserCopyOrMarkData *data)
1566 {
1567         mono_native_tls_set_value (user_copy_or_mark_key, data);
1568 }
1569
1570 static void
1571 single_arg_user_copy_or_mark (void **obj)
1572 {
1573         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
1574
1575         data->func (obj, data->queue);
1576 }
1577
1578 /*
1579  * The memory area from start_root to end_root contains pointers to objects.
1580  * Their position is precisely described by @desc (this means that the pointer
1581  * can be either NULL or the pointer to the start of an object).
1582  * This functions copies them to to_space updates them.
1583  *
1584  * This function is not thread-safe!
1585  */
1586 static void
1587 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1588 {
1589         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1590         SgenGrayQueue *queue = ctx.queue;
1591
1592         switch (desc & ROOT_DESC_TYPE_MASK) {
1593         case ROOT_DESC_BITMAP:
1594                 desc >>= ROOT_DESC_TYPE_SHIFT;
1595                 while (desc) {
1596                         if ((desc & 1) && *start_root) {
1597                                 copy_func (start_root, queue);
1598                                 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1599                                 sgen_drain_gray_stack (-1, ctx);
1600                         }
1601                         desc >>= 1;
1602                         start_root++;
1603                 }
1604                 return;
1605         case ROOT_DESC_COMPLEX: {
1606                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1607                 int bwords = (*bitmap_data) - 1;
1608                 void **start_run = start_root;
1609                 bitmap_data++;
1610                 while (bwords-- > 0) {
1611                         gsize bmap = *bitmap_data++;
1612                         void **objptr = start_run;
1613                         while (bmap) {
1614                                 if ((bmap & 1) && *objptr) {
1615                                         copy_func (objptr, queue);
1616                                         SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1617                                         sgen_drain_gray_stack (-1, ctx);
1618                                 }
1619                                 bmap >>= 1;
1620                                 ++objptr;
1621                         }
1622                         start_run += GC_BITS_PER_WORD;
1623                 }
1624                 break;
1625         }
1626         case ROOT_DESC_USER: {
1627                 UserCopyOrMarkData data = { copy_func, queue };
1628                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1629                 set_user_copy_or_mark_data (&data);
1630                 marker (start_root, single_arg_user_copy_or_mark);
1631                 set_user_copy_or_mark_data (NULL);
1632                 break;
1633         }
1634         case ROOT_DESC_RUN_LEN:
1635                 g_assert_not_reached ();
1636         default:
1637                 g_assert_not_reached ();
1638         }
1639 }
1640
1641 static void
1642 reset_heap_boundaries (void)
1643 {
1644         lowest_heap_address = ~(mword)0;
1645         highest_heap_address = 0;
1646 }
1647
1648 void
1649 sgen_update_heap_boundaries (mword low, mword high)
1650 {
1651         mword old;
1652
1653         do {
1654                 old = lowest_heap_address;
1655                 if (low >= old)
1656                         break;
1657         } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1658
1659         do {
1660                 old = highest_heap_address;
1661                 if (high <= old)
1662                         break;
1663         } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1664 }
1665
1666 /*
1667  * Allocate and setup the data structures needed to be able to allocate objects
1668  * in the nursery. The nursery is stored in nursery_section.
1669  */
1670 static void
1671 alloc_nursery (void)
1672 {
1673         GCMemSection *section;
1674         char *data;
1675         int scan_starts;
1676         int alloc_size;
1677
1678         if (nursery_section)
1679                 return;
1680         SGEN_LOG (2, "Allocating nursery size: %lu", (unsigned long)sgen_nursery_size);
1681         /* later we will alloc a larger area for the nursery but only activate
1682          * what we need. The rest will be used as expansion if we have too many pinned
1683          * objects in the existing nursery.
1684          */
1685         /* FIXME: handle OOM */
1686         section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1687
1688         alloc_size = sgen_nursery_size;
1689
1690         /* If there isn't enough space even for the nursery we should simply abort. */
1691         g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1692
1693 #ifdef SGEN_ALIGN_NURSERY
1694         data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1695 #else
1696         data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1697 #endif
1698         sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1699         SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1700         section->data = section->next_data = data;
1701         section->size = alloc_size;
1702         section->end_data = data + sgen_nursery_size;
1703         scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1704         section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1705         section->num_scan_start = scan_starts;
1706
1707         nursery_section = section;
1708
1709         sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1710 }
1711
1712 void*
1713 mono_gc_get_nursery (int *shift_bits, size_t *size)
1714 {
1715         *size = sgen_nursery_size;
1716 #ifdef SGEN_ALIGN_NURSERY
1717         *shift_bits = DEFAULT_NURSERY_BITS;
1718 #else
1719         *shift_bits = -1;
1720 #endif
1721         return sgen_get_nursery_start ();
1722 }
1723
1724 void
1725 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1726 {
1727         SgenThreadInfo *info = mono_thread_info_current ();
1728
1729         /* Could be called from sgen_thread_unregister () with a NULL info */
1730         if (domain) {
1731                 g_assert (info);
1732                 info->stopped_domain = domain;
1733         }
1734 }
1735
1736 gboolean
1737 mono_gc_precise_stack_mark_enabled (void)
1738 {
1739         return !conservative_stack_mark;
1740 }
1741
1742 FILE *
1743 mono_gc_get_logfile (void)
1744 {
1745         return gc_debug_file;
1746 }
1747
1748 static void
1749 report_finalizer_roots_list (FinalizeReadyEntry *list)
1750 {
1751         GCRootReport report;
1752         FinalizeReadyEntry *fin;
1753
1754         report.count = 0;
1755         for (fin = list; fin; fin = fin->next) {
1756                 if (!fin->object)
1757                         continue;
1758                 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1759         }
1760         notify_gc_roots (&report);
1761 }
1762
1763 static void
1764 report_finalizer_roots (void)
1765 {
1766         report_finalizer_roots_list (fin_ready_list);
1767         report_finalizer_roots_list (critical_fin_list);
1768 }
1769
1770 static GCRootReport *root_report;
1771
1772 static void
1773 single_arg_report_root (void **obj)
1774 {
1775         if (*obj)
1776                 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1777 }
1778
1779 static void
1780 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1781 {
1782         switch (desc & ROOT_DESC_TYPE_MASK) {
1783         case ROOT_DESC_BITMAP:
1784                 desc >>= ROOT_DESC_TYPE_SHIFT;
1785                 while (desc) {
1786                         if ((desc & 1) && *start_root) {
1787                                 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1788                         }
1789                         desc >>= 1;
1790                         start_root++;
1791                 }
1792                 return;
1793         case ROOT_DESC_COMPLEX: {
1794                 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1795                 int bwords = (*bitmap_data) - 1;
1796                 void **start_run = start_root;
1797                 bitmap_data++;
1798                 while (bwords-- > 0) {
1799                         gsize bmap = *bitmap_data++;
1800                         void **objptr = start_run;
1801                         while (bmap) {
1802                                 if ((bmap & 1) && *objptr) {
1803                                         add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1804                                 }
1805                                 bmap >>= 1;
1806                                 ++objptr;
1807                         }
1808                         start_run += GC_BITS_PER_WORD;
1809                 }
1810                 break;
1811         }
1812         case ROOT_DESC_USER: {
1813                 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1814                 root_report = report;
1815                 marker (start_root, single_arg_report_root);
1816                 break;
1817         }
1818         case ROOT_DESC_RUN_LEN:
1819                 g_assert_not_reached ();
1820         default:
1821                 g_assert_not_reached ();
1822         }
1823 }
1824
1825 static void
1826 report_registered_roots_by_type (int root_type)
1827 {
1828         GCRootReport report;
1829         void **start_root;
1830         RootRecord *root;
1831         report.count = 0;
1832         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1833                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1834                 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1835         } SGEN_HASH_TABLE_FOREACH_END;
1836         notify_gc_roots (&report);
1837 }
1838
1839 static void
1840 report_registered_roots (void)
1841 {
1842         report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1843         report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1844 }
1845
1846 static void
1847 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1848 {
1849         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1850         SgenGrayQueue *queue = ctx.queue;
1851         FinalizeReadyEntry *fin;
1852
1853         for (fin = list; fin; fin = fin->next) {
1854                 if (!fin->object)
1855                         continue;
1856                 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1857                 copy_func (&fin->object, queue);
1858         }
1859 }
1860
1861 static const char*
1862 generation_name (int generation)
1863 {
1864         switch (generation) {
1865         case GENERATION_NURSERY: return "nursery";
1866         case GENERATION_OLD: return "old";
1867         default: g_assert_not_reached ();
1868         }
1869 }
1870
1871 const char*
1872 sgen_generation_name (int generation)
1873 {
1874         return generation_name (generation);
1875 }
1876
1877 SgenObjectOperations *
1878 sgen_get_current_object_ops (void){
1879         return &current_object_ops;
1880 }
1881
1882
1883 static void
1884 finish_gray_stack (int generation, GrayQueue *queue)
1885 {
1886         TV_DECLARE (atv);
1887         TV_DECLARE (btv);
1888         int done_with_ephemerons, ephemeron_rounds = 0;
1889         CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1890         ScanObjectFunc scan_func = current_object_ops.scan_object;
1891         ScanCopyContext ctx = { scan_func, copy_func, queue };
1892         char *start_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_start () : NULL;
1893         char *end_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_end () : (char*)-1;
1894
1895         /*
1896          * We copied all the reachable objects. Now it's the time to copy
1897          * the objects that were not referenced by the roots, but by the copied objects.
1898          * we built a stack of objects pointed to by gray_start: they are
1899          * additional roots and we may add more items as we go.
1900          * We loop until gray_start == gray_objects which means no more objects have
1901          * been added. Note this is iterative: no recursion is involved.
1902          * We need to walk the LO list as well in search of marked big objects
1903          * (use a flag since this is needed only on major collections). We need to loop
1904          * here as well, so keep a counter of marked LO (increasing it in copy_object).
1905          *   To achieve better cache locality and cache usage, we drain the gray stack
1906          * frequently, after each object is copied, and just finish the work here.
1907          */
1908         sgen_drain_gray_stack (-1, ctx);
1909         TV_GETTIME (atv);
1910         SGEN_LOG (2, "%s generation done", generation_name (generation));
1911
1912         /*
1913         Reset bridge data, we might have lingering data from a previous collection if this is a major
1914         collection trigged by minor overflow.
1915
1916         We must reset the gathered bridges since their original block might be evacuated due to major
1917         fragmentation in the meanwhile and the bridge code should not have to deal with that.
1918         */
1919         sgen_bridge_reset_data ();
1920
1921         /*
1922          * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1923          * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1924          * objects that are in fact reachable.
1925          */
1926         done_with_ephemerons = 0;
1927         do {
1928                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1929                 sgen_drain_gray_stack (-1, ctx);
1930                 ++ephemeron_rounds;
1931         } while (!done_with_ephemerons);
1932
1933         sgen_scan_togglerefs (start_addr, end_addr, ctx);
1934
1935         if (sgen_need_bridge_processing ()) {
1936                 sgen_collect_bridge_objects (generation, ctx);
1937                 if (generation == GENERATION_OLD)
1938                         sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1939         }
1940
1941         /*
1942         Make sure we drain the gray stack before processing disappearing links and finalizers.
1943         If we don't make sure it is empty we might wrongly see a live object as dead.
1944         */
1945         sgen_drain_gray_stack (-1, ctx);
1946
1947         /*
1948         We must clear weak links that don't track resurrection before processing object ready for
1949         finalization so they can be cleared before that.
1950         */
1951         sgen_null_link_in_range (generation, TRUE, ctx);
1952         if (generation == GENERATION_OLD)
1953                 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1954
1955
1956         /* walk the finalization queue and move also the objects that need to be
1957          * finalized: use the finalized objects as new roots so the objects they depend
1958          * on are also not reclaimed. As with the roots above, only objects in the nursery
1959          * are marked/copied.
1960          */
1961         sgen_finalize_in_range (generation, ctx);
1962         if (generation == GENERATION_OLD)
1963                 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1964         /* drain the new stack that might have been created */
1965         SGEN_LOG (6, "Precise scan of gray area post fin");
1966         sgen_drain_gray_stack (-1, ctx);
1967
1968         /*
1969          * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1970          */
1971         done_with_ephemerons = 0;
1972         do {
1973                 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1974                 sgen_drain_gray_stack (-1, ctx);
1975                 ++ephemeron_rounds;
1976         } while (!done_with_ephemerons);
1977
1978         /*
1979          * Clear ephemeron pairs with unreachable keys.
1980          * We pass the copy func so we can figure out if an array was promoted or not.
1981          */
1982         clear_unreachable_ephemerons (ctx);
1983
1984         TV_GETTIME (btv);
1985         SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
1986
1987         /*
1988          * handle disappearing links
1989          * Note we do this after checking the finalization queue because if an object
1990          * survives (at least long enough to be finalized) we don't clear the link.
1991          * This also deals with a possible issue with the monitor reclamation: with the Boehm
1992          * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1993          * called.
1994          */
1995         g_assert (sgen_gray_object_queue_is_empty (queue));
1996         for (;;) {
1997                 sgen_null_link_in_range (generation, FALSE, ctx);
1998                 if (generation == GENERATION_OLD)
1999                         sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
2000                 if (sgen_gray_object_queue_is_empty (queue))
2001                         break;
2002                 sgen_drain_gray_stack (-1, ctx);
2003         }
2004
2005         g_assert (sgen_gray_object_queue_is_empty (queue));
2006 }
2007
2008 void
2009 sgen_check_section_scan_starts (GCMemSection *section)
2010 {
2011         int i;
2012         for (i = 0; i < section->num_scan_start; ++i) {
2013                 if (section->scan_starts [i]) {
2014                         guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
2015                         g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
2016                 }
2017         }
2018 }
2019
2020 static void
2021 check_scan_starts (void)
2022 {
2023         if (!do_scan_starts_check)
2024                 return;
2025         sgen_check_section_scan_starts (nursery_section);
2026         major_collector.check_scan_starts ();
2027 }
2028
2029 static void
2030 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
2031 {
2032         void **start_root;
2033         RootRecord *root;
2034         SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
2035                 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
2036                 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
2037         } SGEN_HASH_TABLE_FOREACH_END;
2038 }
2039
2040 void
2041 sgen_dump_occupied (char *start, char *end, char *section_start)
2042 {
2043         fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
2044 }
2045
2046 void
2047 sgen_dump_section (GCMemSection *section, const char *type)
2048 {
2049         char *start = section->data;
2050         char *end = section->data + section->size;
2051         char *occ_start = NULL;
2052         GCVTable *vt;
2053         char *old_start = NULL; /* just for debugging */
2054
2055         fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
2056
2057         while (start < end) {
2058                 guint size;
2059                 MonoClass *class;
2060
2061                 if (!*(void**)start) {
2062                         if (occ_start) {
2063                                 sgen_dump_occupied (occ_start, start, section->data);
2064                                 occ_start = NULL;
2065                         }
2066                         start += sizeof (void*); /* should be ALLOC_ALIGN, really */
2067                         continue;
2068                 }
2069                 g_assert (start < section->next_data);
2070
2071                 if (!occ_start)
2072                         occ_start = start;
2073
2074                 vt = (GCVTable*)LOAD_VTABLE (start);
2075                 class = vt->klass;
2076
2077                 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
2078
2079                 /*
2080                 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
2081                                 start - section->data,
2082                                 vt->klass->name_space, vt->klass->name,
2083                                 size);
2084                 */
2085
2086                 old_start = start;
2087                 start += size;
2088         }
2089         if (occ_start)
2090                 sgen_dump_occupied (occ_start, start, section->data);
2091
2092         fprintf (heap_dump_file, "</section>\n");
2093 }
2094
2095 static void
2096 dump_object (MonoObject *obj, gboolean dump_location)
2097 {
2098         static char class_name [1024];
2099
2100         MonoClass *class = mono_object_class (obj);
2101         int i, j;
2102
2103         /*
2104          * Python's XML parser is too stupid to parse angle brackets
2105          * in strings, so we just ignore them;
2106          */
2107         i = j = 0;
2108         while (class->name [i] && j < sizeof (class_name) - 1) {
2109                 if (!strchr ("<>\"", class->name [i]))
2110                         class_name [j++] = class->name [i];
2111                 ++i;
2112         }
2113         g_assert (j < sizeof (class_name));
2114         class_name [j] = 0;
2115
2116         fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
2117                         class->name_space, class_name,
2118                         safe_object_get_size (obj));
2119         if (dump_location) {
2120                 const char *location;
2121                 if (ptr_in_nursery (obj))
2122                         location = "nursery";
2123                 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
2124                         location = "major";
2125                 else
2126                         location = "LOS";
2127                 fprintf (heap_dump_file, " location=\"%s\"", location);
2128         }
2129         fprintf (heap_dump_file, "/>\n");
2130 }
2131
2132 static void
2133 dump_heap (const char *type, int num, const char *reason)
2134 {
2135         ObjectList *list;
2136         LOSObject *bigobj;
2137
2138         fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
2139         if (reason)
2140                 fprintf (heap_dump_file, " reason=\"%s\"", reason);
2141         fprintf (heap_dump_file, ">\n");
2142         fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
2143         sgen_dump_internal_mem_usage (heap_dump_file);
2144         fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
2145         /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
2146         fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
2147
2148         fprintf (heap_dump_file, "<pinned-objects>\n");
2149         for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
2150                 dump_object (list->obj, TRUE);
2151         fprintf (heap_dump_file, "</pinned-objects>\n");
2152
2153         sgen_dump_section (nursery_section, "nursery");
2154
2155         major_collector.dump_heap (heap_dump_file);
2156
2157         fprintf (heap_dump_file, "<los>\n");
2158         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
2159                 dump_object ((MonoObject*)bigobj->data, FALSE);
2160         fprintf (heap_dump_file, "</los>\n");
2161
2162         fprintf (heap_dump_file, "</collection>\n");
2163 }
2164
2165 void
2166 sgen_register_moved_object (void *obj, void *destination)
2167 {
2168         g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
2169
2170         /* FIXME: handle this for parallel collector */
2171         g_assert (!sgen_collection_is_parallel ());
2172
2173         if (moved_objects_idx == MOVED_OBJECTS_NUM) {
2174                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
2175                 moved_objects_idx = 0;
2176         }
2177         moved_objects [moved_objects_idx++] = obj;
2178         moved_objects [moved_objects_idx++] = destination;
2179 }
2180
2181 static void
2182 init_stats (void)
2183 {
2184         static gboolean inited = FALSE;
2185
2186         if (inited)
2187                 return;
2188
2189         mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pre_collection_fragment_clear);
2190         mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_pinning);
2191         mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_remsets);
2192         mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_pinned);
2193         mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_registered_roots);
2194         mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_scan_thread_data);
2195         mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_finish_gray_stack);
2196         mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_minor_fragment_creation);
2197
2198         mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pre_collection_fragment_clear);
2199         mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_pinning);
2200         mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_pinned);
2201         mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_registered_roots);
2202         mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_thread_data);
2203         mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_alloc_pinned);
2204         mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_finalized);
2205         mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_scan_big_objects);
2206         mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_finish_gray_stack);
2207         mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_free_bigobjs);
2208         mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_los_sweep);
2209         mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_sweep);
2210         mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_TIME_INTERVAL, &time_major_fragment_creation);
2211
2212         mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_pinned_objects);
2213
2214 #ifdef HEAVY_STATISTICS
2215         mono_counters_register ("WBarrier remember pointer", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_add_to_global_remset);
2216         mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
2217         mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
2218         mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
2219         mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
2220         mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
2221         mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
2222         mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
2223
2224         mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
2225         mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
2226
2227         mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
2228         mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
2229         mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
2230         mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
2231
2232         mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
2233         mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
2234
2235         mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
2236
2237         mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
2238         mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
2239         mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
2240         mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_to_space);
2241
2242         sgen_nursery_allocator_init_heavy_stats ();
2243         sgen_alloc_init_heavy_stats ();
2244 #endif
2245
2246         inited = TRUE;
2247 }
2248
2249
2250 static void
2251 reset_pinned_from_failed_allocation (void)
2252 {
2253         bytes_pinned_from_failed_allocation = 0;
2254 }
2255
2256 void
2257 sgen_set_pinned_from_failed_allocation (mword objsize)
2258 {
2259         bytes_pinned_from_failed_allocation += objsize;
2260 }
2261
2262 gboolean
2263 sgen_collection_is_parallel (void)
2264 {
2265         switch (current_collection_generation) {
2266         case GENERATION_NURSERY:
2267                 return nursery_collection_is_parallel;
2268         case GENERATION_OLD:
2269                 return major_collector.is_parallel;
2270         default:
2271                 g_error ("Invalid current generation %d", current_collection_generation);
2272         }
2273 }
2274
2275 gboolean
2276 sgen_collection_is_concurrent (void)
2277 {
2278         switch (current_collection_generation) {
2279         case GENERATION_NURSERY:
2280                 return FALSE;
2281         case GENERATION_OLD:
2282                 return concurrent_collection_in_progress;
2283         default:
2284                 g_error ("Invalid current generation %d", current_collection_generation);
2285         }
2286 }
2287
2288 gboolean
2289 sgen_concurrent_collection_in_progress (void)
2290 {
2291         return concurrent_collection_in_progress;
2292 }
2293
2294 typedef struct
2295 {
2296         char *heap_start;
2297         char *heap_end;
2298 } FinishRememberedSetScanJobData;
2299
2300 static void
2301 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2302 {
2303         FinishRememberedSetScanJobData *job_data = job_data_untyped;
2304
2305         remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2306         sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2307 }
2308
2309 typedef struct
2310 {
2311         CopyOrMarkObjectFunc copy_or_mark_func;
2312         ScanObjectFunc scan_func;
2313         char *heap_start;
2314         char *heap_end;
2315         int root_type;
2316 } ScanFromRegisteredRootsJobData;
2317
2318 static void
2319 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2320 {
2321         ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2322         ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2323                 sgen_workers_get_job_gray_queue (worker_data) };
2324
2325         scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2326         sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2327 }
2328
2329 typedef struct
2330 {
2331         char *heap_start;
2332         char *heap_end;
2333 } ScanThreadDataJobData;
2334
2335 static void
2336 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2337 {
2338         ScanThreadDataJobData *job_data = job_data_untyped;
2339
2340         scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2341                         sgen_workers_get_job_gray_queue (worker_data));
2342         sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2343 }
2344
2345 typedef struct
2346 {
2347         FinalizeReadyEntry *list;
2348 } ScanFinalizerEntriesJobData;
2349
2350 static void
2351 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2352 {
2353         ScanFinalizerEntriesJobData *job_data = job_data_untyped;
2354         ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2355
2356         scan_finalizer_entries (job_data->list, ctx);
2357         sgen_free_internal_dynamic (job_data, sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2358 }
2359
2360 static void
2361 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2362 {
2363         g_assert (concurrent_collection_in_progress);
2364         major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2365 }
2366
2367 static void
2368 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2369 {
2370         g_assert (concurrent_collection_in_progress);
2371         sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2372 }
2373
2374 static void
2375 verify_scan_starts (char *start, char *end)
2376 {
2377         int i;
2378
2379         for (i = 0; i < nursery_section->num_scan_start; ++i) {
2380                 char *addr = nursery_section->scan_starts [i];
2381                 if (addr > start && addr < end)
2382                         SGEN_LOG (1, "NFC-BAD SCAN START [%d] %p for obj [%p %p]", i, addr, start, end);
2383         }
2384 }
2385
2386 static void
2387 verify_nursery (void)
2388 {
2389         char *start, *end, *cur, *hole_start;
2390
2391         if (!do_verify_nursery)
2392                 return;
2393
2394         /*This cleans up unused fragments */
2395         sgen_nursery_allocator_prepare_for_pinning ();
2396
2397         hole_start = start = cur = sgen_get_nursery_start ();
2398         end = sgen_get_nursery_end ();
2399
2400         while (cur < end) {
2401                 size_t ss, size;
2402
2403                 if (!*(void**)cur) {
2404                         cur += sizeof (void*);
2405                         continue;
2406                 }
2407
2408                 if (object_is_forwarded (cur))
2409                         SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2410                 else if (object_is_pinned (cur))
2411                         SGEN_LOG (1, "PINNED OBJ %p", cur);
2412
2413                 ss = safe_object_get_size ((MonoObject*)cur);
2414                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2415                 verify_scan_starts (cur, cur + size);
2416                 if (do_dump_nursery_content) {
2417                         if (cur > hole_start)
2418                                 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2419                         SGEN_LOG (1, "OBJ  [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2420                 }
2421                 cur += size;
2422                 hole_start = cur;
2423         }
2424 }
2425
2426 /*
2427  * Checks that no objects in the nursery are fowarded or pinned.  This
2428  * is a precondition to restarting the mutator while doing a
2429  * concurrent collection.  Note that we don't clear fragments because
2430  * we depend on that having happened earlier.
2431  */
2432 static void
2433 check_nursery_is_clean (void)
2434 {
2435         char *start, *end, *cur;
2436
2437         start = cur = sgen_get_nursery_start ();
2438         end = sgen_get_nursery_end ();
2439
2440         while (cur < end) {
2441                 size_t ss, size;
2442
2443                 if (!*(void**)cur) {
2444                         cur += sizeof (void*);
2445                         continue;
2446                 }
2447
2448                 g_assert (!object_is_forwarded (cur));
2449                 g_assert (!object_is_pinned (cur));
2450
2451                 ss = safe_object_get_size ((MonoObject*)cur);
2452                 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2453                 verify_scan_starts (cur, cur + size);
2454
2455                 cur += size;
2456         }
2457 }
2458
2459 static void
2460 init_gray_queue (void)
2461 {
2462         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) {
2463                 sgen_workers_init_distribute_gray_queue ();
2464                 sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
2465                                 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2466         } else {
2467                 sgen_gray_object_queue_init (&gray_queue, NULL);
2468         }
2469 }
2470
2471 static void
2472 pin_stage_object_callback (char *obj, size_t size, void *data)
2473 {
2474         sgen_pin_stage_ptr (obj);
2475         /* FIXME: do pin stats if enabled */
2476 }
2477
2478 /*
2479  * Collect objects in the nursery.  Returns whether to trigger a major
2480  * collection.
2481  */
2482 static gboolean
2483 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2484 {
2485         gboolean needs_major;
2486         size_t max_garbage_amount;
2487         char *nursery_next;
2488         FinishRememberedSetScanJobData *frssjd;
2489         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2490         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2491         ScanThreadDataJobData *stdjd;
2492         mword fragment_total;
2493         ScanCopyContext ctx;
2494         TV_DECLARE (all_atv);
2495         TV_DECLARE (all_btv);
2496         TV_DECLARE (atv);
2497         TV_DECLARE (btv);
2498
2499         if (disable_minor_collections)
2500                 return TRUE;
2501
2502         MONO_GC_BEGIN (GENERATION_NURSERY);
2503         binary_protocol_collection_begin (stat_minor_gcs, GENERATION_NURSERY);
2504
2505         verify_nursery ();
2506
2507 #ifndef DISABLE_PERFCOUNTERS
2508         mono_perfcounters->gc_collections0++;
2509 #endif
2510
2511         current_collection_generation = GENERATION_NURSERY;
2512         if (sgen_collection_is_parallel ())
2513                 current_object_ops = sgen_minor_collector.parallel_ops;
2514         else
2515                 current_object_ops = sgen_minor_collector.serial_ops;
2516
2517         reset_pinned_from_failed_allocation ();
2518
2519         check_scan_starts ();
2520
2521         sgen_nursery_alloc_prepare_for_minor ();
2522
2523         degraded_mode = 0;
2524         objects_pinned = 0;
2525         nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2526         /* FIXME: optimize later to use the higher address where an object can be present */
2527         nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2528
2529         SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", stat_minor_gcs, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2530         max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2531         g_assert (nursery_section->size >= max_garbage_amount);
2532
2533         /* world must be stopped already */
2534         TV_GETTIME (all_atv);
2535         atv = all_atv;
2536
2537         TV_GETTIME (btv);
2538         time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2539
2540         if (xdomain_checks) {
2541                 sgen_clear_nursery_fragments ();
2542                 check_for_xdomain_refs ();
2543         }
2544
2545         nursery_section->next_data = nursery_next;
2546
2547         major_collector.start_nursery_collection ();
2548
2549         sgen_memgov_minor_collection_start ();
2550
2551         init_gray_queue ();
2552
2553         stat_minor_gcs++;
2554         gc_stats.minor_gc_count ++;
2555
2556         MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2557
2558         sgen_process_fin_stage_entries ();
2559         sgen_process_dislink_stage_entries ();
2560
2561         MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2562
2563         /* pin from pinned handles */
2564         sgen_init_pinning ();
2565         mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2566         pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2567         /* pin cemented objects */
2568         sgen_cement_iterate (pin_stage_object_callback, NULL);
2569         /* identify pinned objects */
2570         sgen_optimize_pin_queue (0);
2571         sgen_pinning_setup_section (nursery_section);
2572         ctx.scan_func = NULL;
2573         ctx.copy_func = NULL;
2574         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2575         sgen_pin_objects_in_section (nursery_section, ctx);
2576         sgen_pinning_trim_queue_to_section (nursery_section);
2577
2578         TV_GETTIME (atv);
2579         time_minor_pinning += TV_ELAPSED (btv, atv);
2580         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2581         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2582
2583         MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2584
2585         if (whole_heap_check_before_collection) {
2586                 sgen_clear_nursery_fragments ();
2587                 sgen_check_whole_heap (finish_up_concurrent_mark);
2588         }
2589         if (consistency_check_at_minor_collection)
2590                 sgen_check_consistency ();
2591
2592         sgen_workers_start_all_workers ();
2593         sgen_workers_start_marking ();
2594
2595         frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2596         frssjd->heap_start = sgen_get_nursery_start ();
2597         frssjd->heap_end = nursery_next;
2598         sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2599
2600         /* we don't have complete write barrier yet, so we scan all the old generation sections */
2601         TV_GETTIME (btv);
2602         time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2603         SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2604
2605         MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2606
2607         if (!sgen_collection_is_parallel ()) {
2608                 ctx.scan_func = current_object_ops.scan_object;
2609                 ctx.copy_func = NULL;
2610                 ctx.queue = &gray_queue;
2611                 sgen_drain_gray_stack (-1, ctx);
2612         }
2613
2614         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2615                 report_registered_roots ();
2616         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2617                 report_finalizer_roots ();
2618         TV_GETTIME (atv);
2619         time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2620
2621         MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2622
2623         /* registered roots, this includes static fields */
2624         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2625         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2626         scrrjd_normal->scan_func = current_object_ops.scan_object;
2627         scrrjd_normal->heap_start = sgen_get_nursery_start ();
2628         scrrjd_normal->heap_end = nursery_next;
2629         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2630         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2631
2632         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2633         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2634         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2635         scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2636         scrrjd_wbarrier->heap_end = nursery_next;
2637         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2638         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2639
2640         TV_GETTIME (btv);
2641         time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2642
2643         MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2644
2645         /* thread data */
2646         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2647         stdjd->heap_start = sgen_get_nursery_start ();
2648         stdjd->heap_end = nursery_next;
2649         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2650
2651         TV_GETTIME (atv);
2652         time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2653         btv = atv;
2654
2655         MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2656
2657         g_assert (!sgen_collection_is_parallel () && !sgen_collection_is_concurrent ());
2658
2659         if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ())
2660                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2661
2662         /* Scan the list of objects ready for finalization. If */
2663         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2664         sfejd_fin_ready->list = fin_ready_list;
2665         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
2666
2667         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2668         sfejd_critical_fin->list = critical_fin_list;
2669         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
2670
2671         MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2672
2673         finish_gray_stack (GENERATION_NURSERY, &gray_queue);
2674         TV_GETTIME (atv);
2675         time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2676         mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2677
2678         MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2679
2680         /*
2681          * The (single-threaded) finalization code might have done
2682          * some copying/marking so we can only reset the GC thread's
2683          * worker data here instead of earlier when we joined the
2684          * workers.
2685          */
2686         sgen_workers_reset_data ();
2687
2688         if (objects_pinned) {
2689                 sgen_optimize_pin_queue (0);
2690                 sgen_pinning_setup_section (nursery_section);
2691         }
2692
2693         /* walk the pin_queue, build up the fragment list of free memory, unmark
2694          * pinned objects as we go, memzero() the empty fragments so they are ready for the
2695          * next allocations.
2696          */
2697         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2698         fragment_total = sgen_build_nursery_fragments (nursery_section,
2699                         nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries,
2700                         unpin_queue);
2701         if (!fragment_total)
2702                 degraded_mode = 1;
2703
2704         /* Clear TLABs for all threads */
2705         sgen_clear_tlabs ();
2706
2707         mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2708         TV_GETTIME (btv);
2709         time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2710         SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2711
2712         if (consistency_check_at_minor_collection)
2713                 sgen_check_major_refs ();
2714
2715         major_collector.finish_nursery_collection ();
2716
2717         TV_GETTIME (all_btv);
2718         gc_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
2719
2720         if (heap_dump_file)
2721                 dump_heap ("minor", stat_minor_gcs - 1, NULL);
2722
2723         /* prepare the pin queue for the next collection */
2724         sgen_finish_pinning ();
2725         if (fin_ready_list || critical_fin_list) {
2726                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2727                 mono_gc_finalize_notify ();
2728         }
2729         sgen_pin_stats_reset ();
2730         /* clear cemented hash */
2731         sgen_cement_clear_below_threshold ();
2732
2733         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2734
2735         remset.finish_minor_collection ();
2736
2737         check_scan_starts ();
2738
2739         binary_protocol_flush_buffers (FALSE);
2740
2741         sgen_memgov_minor_collection_end ();
2742
2743         /*objects are late pinned because of lack of memory, so a major is a good call*/
2744         needs_major = objects_pinned > 0;
2745         current_collection_generation = -1;
2746         objects_pinned = 0;
2747
2748         MONO_GC_END (GENERATION_NURSERY);
2749         binary_protocol_collection_end (stat_minor_gcs - 1, GENERATION_NURSERY);
2750
2751         if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2752                 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2753
2754         return needs_major;
2755 }
2756
2757 static void
2758 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2759 {
2760         ctx->scan_func (obj, ctx->queue);
2761 }
2762
2763 static void
2764 scan_nursery_objects (ScanCopyContext ctx)
2765 {
2766         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2767                         (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2768 }
2769
2770 static void
2771 major_copy_or_mark_from_roots (int *old_next_pin_slot, gboolean finish_up_concurrent_mark, gboolean scan_mod_union)
2772 {
2773         LOSObject *bigobj;
2774         TV_DECLARE (atv);
2775         TV_DECLARE (btv);
2776         /* FIXME: only use these values for the precise scan
2777          * note that to_space pointers should be excluded anyway...
2778          */
2779         char *heap_start = NULL;
2780         char *heap_end = (char*)-1;
2781         gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2782         GCRootReport root_report = { 0 };
2783         ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2784         ScanThreadDataJobData *stdjd;
2785         ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2786         ScanCopyContext ctx;
2787
2788         if (concurrent_collection_in_progress) {
2789                 /*This cleans up unused fragments */
2790                 sgen_nursery_allocator_prepare_for_pinning ();
2791
2792                 if (do_concurrent_checks)
2793                         check_nursery_is_clean ();
2794         } else {
2795                 /* The concurrent collector doesn't touch the nursery. */
2796                 sgen_nursery_alloc_prepare_for_major ();
2797         }
2798
2799         init_gray_queue ();
2800
2801         TV_GETTIME (atv);
2802
2803         /* Pinning depends on this */
2804         sgen_clear_nursery_fragments ();
2805
2806         if (whole_heap_check_before_collection)
2807                 sgen_check_whole_heap (finish_up_concurrent_mark);
2808
2809         TV_GETTIME (btv);
2810         time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2811
2812         if (!sgen_collection_is_concurrent ())
2813                 nursery_section->next_data = sgen_get_nursery_end ();
2814         /* we should also coalesce scanning from sections close to each other
2815          * and deal with pointers outside of the sections later.
2816          */
2817
2818         objects_pinned = 0;
2819         *major_collector.have_swept = FALSE;
2820
2821         if (xdomain_checks) {
2822                 sgen_clear_nursery_fragments ();
2823                 check_for_xdomain_refs ();
2824         }
2825
2826         if (!concurrent_collection_in_progress) {
2827                 /* Remsets are not useful for a major collection */
2828                 remset.prepare_for_major_collection ();
2829         }
2830
2831         sgen_process_fin_stage_entries ();
2832         sgen_process_dislink_stage_entries ();
2833
2834         TV_GETTIME (atv);
2835         sgen_init_pinning ();
2836         SGEN_LOG (6, "Collecting pinned addresses");
2837         pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2838
2839         if (!concurrent_collection_in_progress || finish_up_concurrent_mark) {
2840                 if (major_collector.is_concurrent) {
2841                         /*
2842                          * The concurrent major collector cannot evict
2843                          * yet, so we need to pin cemented objects to
2844                          * not break some asserts.
2845                          *
2846                          * FIXME: We could evict now!
2847                          */
2848                         sgen_cement_iterate (pin_stage_object_callback, NULL);
2849                 }
2850
2851                 if (!concurrent_collection_in_progress)
2852                         sgen_cement_reset ();
2853         }
2854
2855         sgen_optimize_pin_queue (0);
2856
2857         /*
2858          * The concurrent collector doesn't move objects, neither on
2859          * the major heap nor in the nursery, so we can mark even
2860          * before pinning has finished.  For the non-concurrent
2861          * collector we start the workers after pinning.
2862          */
2863         if (concurrent_collection_in_progress) {
2864                 sgen_workers_start_all_workers ();
2865                 sgen_workers_start_marking ();
2866         }
2867
2868         /*
2869          * pin_queue now contains all candidate pointers, sorted and
2870          * uniqued.  We must do two passes now to figure out which
2871          * objects are pinned.
2872          *
2873          * The first is to find within the pin_queue the area for each
2874          * section.  This requires that the pin_queue be sorted.  We
2875          * also process the LOS objects and pinned chunks here.
2876          *
2877          * The second, destructive, pass is to reduce the section
2878          * areas to pointers to the actually pinned objects.
2879          */
2880         SGEN_LOG (6, "Pinning from sections");
2881         /* first pass for the sections */
2882         sgen_find_section_pin_queue_start_end (nursery_section);
2883         major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2884         /* identify possible pointers to the insize of large objects */
2885         SGEN_LOG (6, "Pinning from large objects");
2886         for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2887                 int dummy;
2888                 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) {
2889                         binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2890
2891 #ifdef ENABLE_DTRACE
2892                         if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2893                                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2894                                 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2895                         }
2896 #endif
2897
2898                         if (sgen_los_object_is_pinned (bigobj->data)) {
2899                                 g_assert (finish_up_concurrent_mark);
2900                                 continue;
2901                         }
2902                         sgen_los_pin_object (bigobj->data);
2903                         /* FIXME: only enqueue if object has references */
2904                         GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data);
2905                         if (G_UNLIKELY (do_pin_stats))
2906                                 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2907                         SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2908
2909                         if (profile_roots)
2910                                 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2911                 }
2912         }
2913         if (profile_roots)
2914                 notify_gc_roots (&root_report);
2915         /* second pass for the sections */
2916         ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2917         ctx.copy_func = NULL;
2918         ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2919
2920         /*
2921          * Concurrent mark never follows references into the nursery.
2922          * In the start and finish pauses we must scan live nursery
2923          * objects, though.  We could simply scan all nursery objects,
2924          * but that would be conservative.  The easiest way is to do a
2925          * nursery collection, which copies all live nursery objects
2926          * (except pinned ones, with the simple nursery) to the major
2927          * heap.  Scanning the mod union table later will then scan
2928          * those promoted objects, provided they're reachable.  Pinned
2929          * objects in the nursery - which we can trivially find in the
2930          * pinning queue - are treated as roots in the mark pauses.
2931          *
2932          * The split nursery complicates the latter part because
2933          * non-pinned objects can survive in the nursery.  That's why
2934          * we need to do a full front-to-back scan of the nursery,
2935          * marking all objects.
2936          *
2937          * Non-concurrent mark evacuates from the nursery, so it's
2938          * sufficient to just scan pinned nursery objects.
2939          */
2940         if (concurrent_collection_in_progress && sgen_minor_collector.is_split) {
2941                 scan_nursery_objects (ctx);
2942         } else {
2943                 sgen_pin_objects_in_section (nursery_section, ctx);
2944                 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2945                         sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2946         }
2947
2948         major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2949         if (old_next_pin_slot)
2950                 *old_next_pin_slot = sgen_get_pinned_count ();
2951
2952         TV_GETTIME (btv);
2953         time_major_pinning += TV_ELAPSED (atv, btv);
2954         SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2955         SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ());
2956
2957         major_collector.init_to_space ();
2958
2959 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2960         main_gc_thread = mono_native_thread_self ();
2961 #endif
2962
2963         if (!concurrent_collection_in_progress && major_collector.is_parallel) {
2964                 sgen_workers_start_all_workers ();
2965                 sgen_workers_start_marking ();
2966         }
2967
2968         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2969                 report_registered_roots ();
2970         TV_GETTIME (atv);
2971         time_major_scan_pinned += TV_ELAPSED (btv, atv);
2972
2973         /* registered roots, this includes static fields */
2974         scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2975         scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2976         scrrjd_normal->scan_func = current_object_ops.scan_object;
2977         scrrjd_normal->heap_start = heap_start;
2978         scrrjd_normal->heap_end = heap_end;
2979         scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2980         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2981
2982         scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2983         scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2984         scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2985         scrrjd_wbarrier->heap_start = heap_start;
2986         scrrjd_wbarrier->heap_end = heap_end;
2987         scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2988         sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2989
2990         TV_GETTIME (btv);
2991         time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
2992
2993         /* Threads */
2994         stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2995         stdjd->heap_start = heap_start;
2996         stdjd->heap_end = heap_end;
2997         sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2998
2999         TV_GETTIME (atv);
3000         time_major_scan_thread_data += TV_ELAPSED (btv, atv);
3001
3002         TV_GETTIME (btv);
3003         time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
3004
3005         if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
3006                 report_finalizer_roots ();
3007
3008         /* scan the list of objects ready for finalization */
3009         sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3010         sfejd_fin_ready->list = fin_ready_list;
3011         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
3012
3013         sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
3014         sfejd_critical_fin->list = critical_fin_list;
3015         sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
3016
3017         if (scan_mod_union) {
3018                 g_assert (finish_up_concurrent_mark);
3019
3020                 /* Mod union card table */
3021                 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
3022                 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
3023         }
3024
3025         TV_GETTIME (atv);
3026         time_major_scan_finalized += TV_ELAPSED (btv, atv);
3027         SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
3028
3029         TV_GETTIME (btv);
3030         time_major_scan_big_objects += TV_ELAPSED (atv, btv);
3031
3032         if (concurrent_collection_in_progress) {
3033                 /* prepare the pin queue for the next collection */
3034                 sgen_finish_pinning ();
3035
3036                 sgen_pin_stats_reset ();
3037
3038                 if (do_concurrent_checks)
3039                         check_nursery_is_clean ();
3040         }
3041 }
3042
3043 static void
3044 major_start_collection (gboolean concurrent, int *old_next_pin_slot)
3045 {
3046         MONO_GC_BEGIN (GENERATION_OLD);
3047         binary_protocol_collection_begin (stat_major_gcs, GENERATION_OLD);
3048
3049         current_collection_generation = GENERATION_OLD;
3050 #ifndef DISABLE_PERFCOUNTERS
3051         mono_perfcounters->gc_collections1++;
3052 #endif
3053
3054         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3055
3056         if (concurrent) {
3057                 g_assert (major_collector.is_concurrent);
3058                 concurrent_collection_in_progress = TRUE;
3059
3060                 sgen_cement_concurrent_start ();
3061
3062                 current_object_ops = major_collector.major_concurrent_ops;
3063         } else {
3064                 current_object_ops = major_collector.major_ops;
3065         }
3066
3067         reset_pinned_from_failed_allocation ();
3068
3069         sgen_memgov_major_collection_start ();
3070
3071         //count_ref_nonref_objs ();
3072         //consistency_check ();
3073
3074         check_scan_starts ();
3075
3076         degraded_mode = 0;
3077         SGEN_LOG (1, "Start major collection %d", stat_major_gcs);
3078         stat_major_gcs++;
3079         gc_stats.major_gc_count ++;
3080
3081         if (major_collector.start_major_collection)
3082                 major_collector.start_major_collection ();
3083
3084         major_copy_or_mark_from_roots (old_next_pin_slot, FALSE, FALSE);
3085 }
3086
3087 static void
3088 wait_for_workers_to_finish (void)
3089 {
3090         if (concurrent_collection_in_progress || major_collector.is_parallel) {
3091                 gray_queue_redirect (&gray_queue);
3092                 sgen_workers_join ();
3093         }
3094
3095         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3096
3097 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
3098         main_gc_thread = NULL;
3099 #endif
3100 }
3101
3102 static void
3103 major_finish_collection (const char *reason, int old_next_pin_slot, gboolean scan_mod_union)
3104 {
3105         LOSObject *bigobj, *prevbo;
3106         TV_DECLARE (atv);
3107         TV_DECLARE (btv);
3108
3109         TV_GETTIME (btv);
3110
3111         if (concurrent_collection_in_progress || major_collector.is_parallel)
3112                 wait_for_workers_to_finish ();
3113
3114         if (concurrent_collection_in_progress) {
3115                 current_object_ops = major_collector.major_concurrent_ops;
3116
3117                 major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
3118                 wait_for_workers_to_finish ();
3119
3120                 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3121
3122                 if (do_concurrent_checks)
3123                         check_nursery_is_clean ();
3124         } else {
3125                 current_object_ops = major_collector.major_ops;
3126         }
3127
3128         /*
3129          * The workers have stopped so we need to finish gray queue
3130          * work that might result from finalization in the main GC
3131          * thread.  Redirection must therefore be turned off.
3132          */
3133         sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
3134         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3135
3136         /* all the objects in the heap */
3137         finish_gray_stack (GENERATION_OLD, &gray_queue);
3138         TV_GETTIME (atv);
3139         time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
3140
3141         /*
3142          * The (single-threaded) finalization code might have done
3143          * some copying/marking so we can only reset the GC thread's
3144          * worker data here instead of earlier when we joined the
3145          * workers.
3146          */
3147         sgen_workers_reset_data ();
3148
3149         if (objects_pinned) {
3150                 g_assert (!concurrent_collection_in_progress);
3151
3152                 /*This is slow, but we just OOM'd*/
3153                 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
3154                 sgen_optimize_pin_queue (0);
3155                 sgen_find_section_pin_queue_start_end (nursery_section);
3156                 objects_pinned = 0;
3157         }
3158
3159         reset_heap_boundaries ();
3160         sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
3161
3162         if (check_mark_bits_after_major_collection)
3163                 sgen_check_major_heap_marked ();
3164
3165         MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
3166
3167         /* sweep the big objects list */
3168         prevbo = NULL;
3169         for (bigobj = los_object_list; bigobj;) {
3170                 g_assert (!object_is_pinned (bigobj->data));
3171                 if (sgen_los_object_is_pinned (bigobj->data)) {
3172                         sgen_los_unpin_object (bigobj->data);
3173                         sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
3174                 } else {
3175                         LOSObject *to_free;
3176                         /* not referenced anywhere, so we can free it */
3177                         if (prevbo)
3178                                 prevbo->next = bigobj->next;
3179                         else
3180                                 los_object_list = bigobj->next;
3181                         to_free = bigobj;
3182                         bigobj = bigobj->next;
3183                         sgen_los_free_object (to_free);
3184                         continue;
3185                 }
3186                 prevbo = bigobj;
3187                 bigobj = bigobj->next;
3188         }
3189
3190         TV_GETTIME (btv);
3191         time_major_free_bigobjs += TV_ELAPSED (atv, btv);
3192
3193         sgen_los_sweep ();
3194
3195         TV_GETTIME (atv);
3196         time_major_los_sweep += TV_ELAPSED (btv, atv);
3197
3198         major_collector.sweep ();
3199
3200         MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
3201
3202         TV_GETTIME (btv);
3203         time_major_sweep += TV_ELAPSED (atv, btv);
3204
3205         if (!concurrent_collection_in_progress) {
3206                 /* walk the pin_queue, build up the fragment list of free memory, unmark
3207                  * pinned objects as we go, memzero() the empty fragments so they are ready for the
3208                  * next allocations.
3209                  */
3210                 if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries, NULL))
3211                         degraded_mode = 1;
3212
3213                 /* prepare the pin queue for the next collection */
3214                 sgen_finish_pinning ();
3215
3216                 /* Clear TLABs for all threads */
3217                 sgen_clear_tlabs ();
3218
3219                 sgen_pin_stats_reset ();
3220         }
3221
3222         if (concurrent_collection_in_progress)
3223                 sgen_cement_concurrent_finish ();
3224         sgen_cement_clear_below_threshold ();
3225
3226         TV_GETTIME (atv);
3227         time_major_fragment_creation += TV_ELAPSED (btv, atv);
3228
3229         if (heap_dump_file)
3230                 dump_heap ("major", stat_major_gcs - 1, reason);
3231
3232         if (fin_ready_list || critical_fin_list) {
3233                 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
3234                 mono_gc_finalize_notify ();
3235         }
3236
3237         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3238
3239         sgen_memgov_major_collection_end ();
3240         current_collection_generation = -1;
3241
3242         major_collector.finish_major_collection ();
3243
3244         g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
3245
3246         if (concurrent_collection_in_progress)
3247                 concurrent_collection_in_progress = FALSE;
3248
3249         check_scan_starts ();
3250
3251         binary_protocol_flush_buffers (FALSE);
3252
3253         //consistency_check ();
3254
3255         MONO_GC_END (GENERATION_OLD);
3256         binary_protocol_collection_end (stat_major_gcs - 1, GENERATION_OLD);
3257 }
3258
3259 static gboolean
3260 major_do_collection (const char *reason)
3261 {
3262         TV_DECLARE (all_atv);
3263         TV_DECLARE (all_btv);
3264         int old_next_pin_slot;
3265
3266         if (major_collector.get_and_reset_num_major_objects_marked) {
3267                 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
3268                 g_assert (!num_marked);
3269         }
3270
3271         /* world must be stopped already */
3272         TV_GETTIME (all_atv);
3273
3274         major_start_collection (FALSE, &old_next_pin_slot);
3275         major_finish_collection (reason, old_next_pin_slot, FALSE);
3276
3277         TV_GETTIME (all_btv);
3278         gc_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
3279
3280         /* FIXME: also report this to the user, preferably in gc-end. */
3281         if (major_collector.get_and_reset_num_major_objects_marked)
3282                 major_collector.get_and_reset_num_major_objects_marked ();
3283
3284         return bytes_pinned_from_failed_allocation > 0;
3285 }
3286
3287 static gboolean major_do_collection (const char *reason);
3288
3289 static void
3290 major_start_concurrent_collection (const char *reason)
3291 {
3292         long long num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3293
3294         g_assert (num_objects_marked == 0);
3295
3296         MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
3297
3298         // FIXME: store reason and pass it when finishing
3299         major_start_collection (TRUE, NULL);
3300
3301         gray_queue_redirect (&gray_queue);
3302         sgen_workers_wait_for_jobs ();
3303
3304         num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3305         MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
3306
3307         current_collection_generation = -1;
3308 }
3309
3310 static gboolean
3311 major_update_or_finish_concurrent_collection (gboolean force_finish)
3312 {
3313         SgenGrayQueue unpin_queue;
3314         memset (&unpin_queue, 0, sizeof (unpin_queue));
3315
3316         MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3317
3318         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3319
3320         major_collector.update_cardtable_mod_union ();
3321         sgen_los_update_cardtable_mod_union ();
3322
3323         if (!force_finish && !sgen_workers_all_done ()) {
3324                 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3325                 return FALSE;
3326         }
3327
3328         if (mod_union_consistency_check)
3329                 sgen_check_mod_union_consistency ();
3330
3331         collect_nursery (&unpin_queue, TRUE);
3332
3333         current_collection_generation = GENERATION_OLD;
3334         major_finish_collection ("finishing", -1, TRUE);
3335
3336         if (whole_heap_check_before_collection)
3337                 sgen_check_whole_heap (FALSE);
3338
3339         unpin_objects_from_queue (&unpin_queue);
3340         sgen_gray_object_queue_deinit (&unpin_queue);
3341
3342         MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3343
3344         current_collection_generation = -1;
3345
3346         return TRUE;
3347 }
3348
3349 /*
3350  * Ensure an allocation request for @size will succeed by freeing enough memory.
3351  *
3352  * LOCKING: The GC lock MUST be held.
3353  */
3354 void
3355 sgen_ensure_free_space (size_t size)
3356 {
3357         int generation_to_collect = -1;
3358         const char *reason = NULL;
3359
3360
3361         if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3362                 if (sgen_need_major_collection (size)) {
3363                         reason = "LOS overflow";
3364                         generation_to_collect = GENERATION_OLD;
3365                 }
3366         } else {
3367                 if (degraded_mode) {
3368                         if (sgen_need_major_collection (size)) {
3369                                 reason = "Degraded mode overflow";
3370                                 generation_to_collect = GENERATION_OLD;
3371                         }
3372                 } else if (sgen_need_major_collection (size)) {
3373                         reason = "Minor allowance";
3374                         generation_to_collect = GENERATION_OLD;
3375                 } else {
3376                         generation_to_collect = GENERATION_NURSERY;
3377                         reason = "Nursery full";
3378                 }
3379         }
3380
3381         if (generation_to_collect == -1) {
3382                 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3383                         generation_to_collect = GENERATION_OLD;
3384                         reason = "Finish concurrent collection";
3385                 }
3386         }
3387
3388         if (generation_to_collect == -1)
3389                 return;
3390         sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3391 }
3392
3393 /*
3394  * LOCKING: Assumes the GC lock is held.
3395  */
3396 void
3397 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3398 {
3399         TV_DECLARE (gc_end);
3400         GGTimingInfo infos [2];
3401         int overflow_generation_to_collect = -1;
3402         int oldest_generation_collected = generation_to_collect;
3403         const char *overflow_reason = NULL;
3404
3405         MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3406         if (wait_to_finish)
3407                 binary_protocol_collection_force (generation_to_collect);
3408
3409         g_assert (generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD);
3410
3411         memset (infos, 0, sizeof (infos));
3412         mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3413
3414         infos [0].generation = generation_to_collect;
3415         infos [0].reason = reason;
3416         infos [0].is_overflow = FALSE;
3417         TV_GETTIME (infos [0].total_time);
3418         infos [1].generation = -1;
3419
3420         sgen_stop_world (generation_to_collect);
3421
3422         if (concurrent_collection_in_progress) {
3423                 if (major_update_or_finish_concurrent_collection (wait_to_finish && generation_to_collect == GENERATION_OLD)) {
3424                         oldest_generation_collected = GENERATION_OLD;
3425                         goto done;
3426                 }
3427                 if (generation_to_collect == GENERATION_OLD)
3428                         goto done;
3429         } else {
3430                 if (generation_to_collect == GENERATION_OLD &&
3431                                 allow_synchronous_major &&
3432                                 major_collector.want_synchronous_collection &&
3433                                 *major_collector.want_synchronous_collection) {
3434                         wait_to_finish = TRUE;
3435                 }
3436         }
3437
3438         //FIXME extract overflow reason
3439         if (generation_to_collect == GENERATION_NURSERY) {
3440                 if (collect_nursery (NULL, FALSE)) {
3441                         overflow_generation_to_collect = GENERATION_OLD;
3442                         overflow_reason = "Minor overflow";
3443                 }
3444         } else {
3445                 if (major_collector.is_concurrent) {
3446                         g_assert (!concurrent_collection_in_progress);
3447                         if (!wait_to_finish)
3448                                 collect_nursery (NULL, FALSE);
3449                 }
3450
3451                 if (major_collector.is_concurrent && !wait_to_finish) {
3452                         major_start_concurrent_collection (reason);
3453                         // FIXME: set infos[0] properly
3454                         goto done;
3455                 } else {
3456                         if (major_do_collection (reason)) {
3457                                 overflow_generation_to_collect = GENERATION_NURSERY;
3458                                 overflow_reason = "Excessive pinning";
3459                         }
3460                 }
3461         }
3462
3463         TV_GETTIME (gc_end);
3464         infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end);
3465
3466
3467         if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) {
3468                 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3469                 infos [1].generation = overflow_generation_to_collect;
3470                 infos [1].reason = overflow_reason;
3471                 infos [1].is_overflow = TRUE;
3472                 infos [1].total_time = gc_end;
3473
3474                 if (overflow_generation_to_collect == GENERATION_NURSERY)
3475                         collect_nursery (NULL, FALSE);
3476                 else
3477                         major_do_collection (overflow_reason);
3478
3479                 TV_GETTIME (gc_end);
3480                 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3481
3482                 /* keep events symmetric */
3483                 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3484
3485                 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3486         }
3487
3488         SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3489
3490         /* this also sets the proper pointers for the next allocation */
3491         if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3492                 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3493                 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%d pinned)", requested_size, sgen_get_pinned_count ());
3494                 sgen_dump_pin_queue ();
3495                 degraded_mode = 1;
3496         }
3497
3498  done:
3499         g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3500
3501         sgen_restart_world (oldest_generation_collected, infos);
3502
3503         mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3504 }
3505
3506 /*
3507  * ######################################################################
3508  * ########  Memory allocation from the OS
3509  * ######################################################################
3510  * This section of code deals with getting memory from the OS and
3511  * allocating memory for GC-internal data structures.
3512  * Internal memory can be handled with a freelist for small objects.
3513  */
3514
3515 /*
3516  * Debug reporting.
3517  */
3518 G_GNUC_UNUSED static void
3519 report_internal_mem_usage (void)
3520 {
3521         printf ("Internal memory usage:\n");
3522         sgen_report_internal_mem_usage ();
3523         printf ("Pinned memory usage:\n");
3524         major_collector.report_pinned_memory_usage ();
3525 }
3526
3527 /*
3528  * ######################################################################
3529  * ########  Finalization support
3530  * ######################################################################
3531  */
3532
3533 static inline gboolean
3534 sgen_major_is_object_alive (void *object)
3535 {
3536         mword objsize;
3537
3538         /* Oldgen objects can be pinned and forwarded too */
3539         if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3540                 return TRUE;
3541
3542         /*
3543          * FIXME: major_collector.is_object_live() also calculates the
3544          * size.  Avoid the double calculation.
3545          */
3546         objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3547         if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3548                 return sgen_los_object_is_pinned (object);
3549
3550         return major_collector.is_object_live (object);
3551 }
3552
3553 /*
3554  * If the object has been forwarded it means it's still referenced from a root.
3555  * If it is pinned it's still alive as well.
3556  * A LOS object is only alive if we have pinned it.
3557  * Return TRUE if @obj is ready to be finalized.
3558  */
3559 static inline gboolean
3560 sgen_is_object_alive (void *object)
3561 {
3562         if (ptr_in_nursery (object))
3563                 return sgen_nursery_is_object_alive (object);
3564
3565         return sgen_major_is_object_alive (object);
3566 }
3567
3568 /*
3569  * This function returns true if @object is either alive or it belongs to the old gen
3570  * and we're currently doing a minor collection.
3571  */
3572 static inline int
3573 sgen_is_object_alive_for_current_gen (char *object)
3574 {
3575         if (ptr_in_nursery (object))
3576                 return sgen_nursery_is_object_alive (object);
3577
3578         if (current_collection_generation == GENERATION_NURSERY)
3579                 return TRUE;
3580
3581         return sgen_major_is_object_alive (object);
3582 }
3583
3584 /*
3585  * This function returns true if @object is either alive and belongs to the
3586  * current collection - major collections are full heap, so old gen objects
3587  * are never alive during a minor collection.
3588  */
3589 static inline int
3590 sgen_is_object_alive_and_on_current_collection (char *object)
3591 {
3592         if (ptr_in_nursery (object))
3593                 return sgen_nursery_is_object_alive (object);
3594
3595         if (current_collection_generation == GENERATION_NURSERY)
3596                 return FALSE;
3597
3598         return sgen_major_is_object_alive (object);
3599 }
3600
3601
3602 gboolean
3603 sgen_gc_is_object_ready_for_finalization (void *object)
3604 {
3605         return !sgen_is_object_alive (object);
3606 }
3607
3608 static gboolean
3609 has_critical_finalizer (MonoObject *obj)
3610 {
3611         MonoClass *class;
3612
3613         if (!mono_defaults.critical_finalizer_object)
3614                 return FALSE;
3615
3616         class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3617
3618         return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3619 }
3620
3621 void
3622 sgen_queue_finalization_entry (MonoObject *obj)
3623 {
3624         FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3625         gboolean critical = has_critical_finalizer (obj);
3626         entry->object = obj;
3627         if (critical) {
3628                 entry->next = critical_fin_list;
3629                 critical_fin_list = entry;
3630         } else {
3631                 entry->next = fin_ready_list;
3632                 fin_ready_list = entry;
3633         }
3634
3635 #ifdef ENABLE_DTRACE
3636         if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3637                 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3638                 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3639                 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3640                                 vt->klass->name_space, vt->klass->name, gen, critical);
3641         }
3642 #endif
3643 }
3644
3645 gboolean
3646 sgen_object_is_live (void *obj)
3647 {
3648         return sgen_is_object_alive_and_on_current_collection (obj);
3649 }
3650
3651 /* LOCKING: requires that the GC lock is held */
3652 static void
3653 null_ephemerons_for_domain (MonoDomain *domain)
3654 {
3655         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3656
3657         while (current) {
3658                 MonoObject *object = (MonoObject*)current->array;
3659
3660                 if (object && !object->vtable) {
3661                         EphemeronLinkNode *tmp = current;
3662
3663                         if (prev)
3664                                 prev->next = current->next;
3665                         else
3666                                 ephemeron_list = current->next;
3667
3668                         current = current->next;
3669                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3670                 } else {
3671                         prev = current;
3672                         current = current->next;
3673                 }
3674         }
3675 }
3676
3677 /* LOCKING: requires that the GC lock is held */
3678 static void
3679 clear_unreachable_ephemerons (ScanCopyContext ctx)
3680 {
3681         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3682         GrayQueue *queue = ctx.queue;
3683         EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3684         MonoArray *array;
3685         Ephemeron *cur, *array_end;
3686         char *tombstone;
3687
3688         while (current) {
3689                 char *object = current->array;
3690
3691                 if (!sgen_is_object_alive_for_current_gen (object)) {
3692                         EphemeronLinkNode *tmp = current;
3693
3694                         SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3695
3696                         if (prev)
3697                                 prev->next = current->next;
3698                         else
3699                                 ephemeron_list = current->next;
3700
3701                         current = current->next;
3702                         sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3703
3704                         continue;
3705                 }
3706
3707                 copy_func ((void**)&object, queue);
3708                 current->array = object;
3709
3710                 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3711
3712                 array = (MonoArray*)object;
3713                 cur = mono_array_addr (array, Ephemeron, 0);
3714                 array_end = cur + mono_array_length_fast (array);
3715                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3716
3717                 for (; cur < array_end; ++cur) {
3718                         char *key = (char*)cur->key;
3719
3720                         if (!key || key == tombstone)
3721                                 continue;
3722
3723                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3724                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3725                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3726
3727                         if (!sgen_is_object_alive_for_current_gen (key)) {
3728                                 cur->key = tombstone;
3729                                 cur->value = NULL;
3730                                 continue;
3731                         }
3732                 }
3733                 prev = current;
3734                 current = current->next;
3735         }
3736 }
3737
3738 /*
3739 LOCKING: requires that the GC lock is held
3740
3741 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3742 */
3743 static int
3744 mark_ephemerons_in_range (ScanCopyContext ctx)
3745 {
3746         CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3747         GrayQueue *queue = ctx.queue;
3748         int nothing_marked = 1;
3749         EphemeronLinkNode *current = ephemeron_list;
3750         MonoArray *array;
3751         Ephemeron *cur, *array_end;
3752         char *tombstone;
3753
3754         for (current = ephemeron_list; current; current = current->next) {
3755                 char *object = current->array;
3756                 SGEN_LOG (5, "Ephemeron array at %p", object);
3757
3758                 /*It has to be alive*/
3759                 if (!sgen_is_object_alive_for_current_gen (object)) {
3760                         SGEN_LOG (5, "\tnot reachable");
3761                         continue;
3762                 }
3763
3764                 copy_func ((void**)&object, queue);
3765
3766                 array = (MonoArray*)object;
3767                 cur = mono_array_addr (array, Ephemeron, 0);
3768                 array_end = cur + mono_array_length_fast (array);
3769                 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3770
3771                 for (; cur < array_end; ++cur) {
3772                         char *key = cur->key;
3773
3774                         if (!key || key == tombstone)
3775                                 continue;
3776
3777                         SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3778                                 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3779                                 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3780
3781                         if (sgen_is_object_alive_for_current_gen (key)) {
3782                                 char *value = cur->value;
3783
3784                                 copy_func ((void**)&cur->key, queue);
3785                                 if (value) {
3786                                         if (!sgen_is_object_alive_for_current_gen (value))
3787                                                 nothing_marked = 0;
3788                                         copy_func ((void**)&cur->value, queue);
3789                                 }
3790                         }
3791                 }
3792         }
3793
3794         SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3795         return nothing_marked;
3796 }
3797
3798 int
3799 mono_gc_invoke_finalizers (void)
3800 {
3801         FinalizeReadyEntry *entry = NULL;
3802         gboolean entry_is_critical = FALSE;
3803         int count = 0;
3804         void *obj;
3805         /* FIXME: batch to reduce lock contention */
3806         while (fin_ready_list || critical_fin_list) {
3807                 LOCK_GC;
3808
3809                 if (entry) {
3810                         FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3811
3812                         /* We have finalized entry in the last
3813                            interation, now we need to remove it from
3814                            the list. */
3815                         if (*list == entry)
3816                                 *list = entry->next;
3817                         else {
3818                                 FinalizeReadyEntry *e = *list;
3819                                 while (e->next != entry)
3820                                         e = e->next;
3821                                 e->next = entry->next;
3822                         }
3823                         sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3824                         entry = NULL;
3825                 }
3826
3827                 /* Now look for the first non-null entry. */
3828                 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3829                         ;
3830                 if (entry) {
3831                         entry_is_critical = FALSE;
3832                 } else {
3833                         entry_is_critical = TRUE;
3834                         for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3835                                 ;
3836                 }
3837
3838                 if (entry) {
3839                         g_assert (entry->object);
3840                         num_ready_finalizers--;
3841                         obj = entry->object;
3842                         entry->object = NULL;
3843                         SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3844                 }
3845
3846                 UNLOCK_GC;
3847
3848                 if (!entry)
3849                         break;
3850
3851                 g_assert (entry->object == NULL);
3852                 count++;
3853                 /* the object is on the stack so it is pinned */
3854                 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3855                 mono_gc_run_finalize (obj, NULL);
3856         }
3857         g_assert (!entry);
3858         return count;
3859 }
3860
3861 gboolean
3862 mono_gc_pending_finalizers (void)
3863 {
3864         return fin_ready_list || critical_fin_list;
3865 }
3866
3867 /*
3868  * ######################################################################
3869  * ########  registered roots support
3870  * ######################################################################
3871  */
3872
3873 /*
3874  * We do not coalesce roots.
3875  */
3876 static int
3877 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3878 {
3879         RootRecord new_root;
3880         int i;
3881         LOCK_GC;
3882         for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3883                 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3884                 /* we allow changing the size and the descriptor (for thread statics etc) */
3885                 if (root) {
3886                         size_t old_size = root->end_root - start;
3887                         root->end_root = start + size;
3888                         g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3889                                           ((root->root_desc == 0) && (descr == NULL)));
3890                         root->root_desc = (mword)descr;
3891                         roots_size += size;
3892                         roots_size -= old_size;
3893                         UNLOCK_GC;
3894                         return TRUE;
3895                 }
3896         }
3897
3898         new_root.end_root = start + size;
3899         new_root.root_desc = (mword)descr;
3900
3901         sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3902         roots_size += size;
3903
3904         SGEN_LOG (3, "Added root for range: %p-%p, descr: %p  (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3905
3906         UNLOCK_GC;
3907         return TRUE;
3908 }
3909
3910 int
3911 mono_gc_register_root (char *start, size_t size, void *descr)
3912 {
3913         return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3914 }
3915
3916 int
3917 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3918 {
3919         return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3920 }
3921
3922 void
3923 mono_gc_deregister_root (char* addr)
3924 {
3925         int root_type;
3926         RootRecord root;
3927
3928         LOCK_GC;
3929         for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3930                 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3931                         roots_size -= (root.end_root - addr);
3932         }
3933         UNLOCK_GC;
3934 }
3935
3936 /*
3937  * ######################################################################
3938  * ########  Thread handling (stop/start code)
3939  * ######################################################################
3940  */
3941
3942 unsigned int sgen_global_stop_count = 0;
3943
3944 int
3945 sgen_get_current_collection_generation (void)
3946 {
3947         return current_collection_generation;
3948 }
3949
3950 void
3951 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3952 {
3953         gc_callbacks = *callbacks;
3954 }
3955
3956 MonoGCCallbacks *
3957 mono_gc_get_gc_callbacks ()
3958 {
3959         return &gc_callbacks;
3960 }
3961
3962 /* Variables holding start/end nursery so it won't have to be passed at every call */
3963 static void *scan_area_arg_start, *scan_area_arg_end;
3964
3965 void
3966 mono_gc_conservatively_scan_area (void *start, void *end)
3967 {
3968         conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
3969 }
3970
3971 void*
3972 mono_gc_scan_object (void *obj)
3973 {
3974         UserCopyOrMarkData *data = mono_native_tls_get_value (user_copy_or_mark_key);
3975         current_object_ops.copy_or_mark_object (&obj, data->queue);
3976         return obj;
3977 }
3978
3979 /*
3980  * Mark from thread stacks and registers.
3981  */
3982 static void
3983 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
3984 {
3985         SgenThreadInfo *info;
3986
3987         scan_area_arg_start = start_nursery;
3988         scan_area_arg_end = end_nursery;
3989
3990         FOREACH_THREAD (info) {
3991                 if (info->skip) {
3992                         SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3993                         continue;
3994                 }
3995                 if (info->gc_disabled) {
3996                         SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3997                         continue;
3998                 }
3999
4000                 if (!info->joined_stw) {
4001                         SGEN_LOG (3, "Skipping thread not seen in STW %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
4002                         continue;
4003                 }
4004
4005                 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
4006                 if (!info->thread_is_dying) {
4007                         if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
4008                                 UserCopyOrMarkData data = { NULL, queue };
4009                                 set_user_copy_or_mark_data (&data);
4010                                 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise);
4011                                 set_user_copy_or_mark_data (NULL);
4012                         } else if (!precise) {
4013                                 if (!conservative_stack_mark) {
4014                                         fprintf (stderr, "Precise stack mark not supported - disabling.\n");
4015                                         conservative_stack_mark = TRUE;
4016                                 }
4017                                 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
4018                         }
4019                 }
4020
4021                 if (!info->thread_is_dying && !precise) {
4022 #ifdef USE_MONO_CTX
4023                         conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
4024                                 start_nursery, end_nursery, PIN_TYPE_STACK);
4025 #else
4026                         conservatively_pin_objects_from (&info->regs, &info->regs + ARCH_NUM_REGS,
4027                                         start_nursery, end_nursery, PIN_TYPE_STACK);
4028 #endif
4029                 }
4030         } END_FOREACH_THREAD
4031 }
4032
4033 static gboolean
4034 ptr_on_stack (void *ptr)
4035 {
4036         gpointer stack_start = &stack_start;
4037         SgenThreadInfo *info = mono_thread_info_current ();
4038
4039         if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
4040                 return TRUE;
4041         return FALSE;
4042 }
4043
4044 static void*
4045 sgen_thread_register (SgenThreadInfo* info, void *addr)
4046 {
4047         LOCK_GC;
4048 #ifndef HAVE_KW_THREAD
4049         info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
4050
4051         g_assert (!mono_native_tls_get_value (thread_info_key));
4052         mono_native_tls_set_value (thread_info_key, info);
4053 #else
4054         sgen_thread_info = info;
4055 #endif
4056
4057 #if !defined(__MACH__)
4058         info->stop_count = -1;
4059         info->signal = 0;
4060 #endif
4061         info->skip = 0;
4062         info->joined_stw = FALSE;
4063         info->doing_handshake = FALSE;
4064         info->thread_is_dying = FALSE;
4065         info->stack_start = NULL;
4066         info->stopped_ip = NULL;
4067         info->stopped_domain = NULL;
4068 #ifdef USE_MONO_CTX
4069         memset (&info->ctx, 0, sizeof (MonoContext));
4070 #else
4071         memset (&info->regs, 0, sizeof (info->regs));
4072 #endif
4073
4074         sgen_init_tlab_info (info);
4075
4076         binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
4077
4078         /* try to get it with attributes first */
4079 #if (defined(HAVE_PTHREAD_GETATTR_NP) || defined(HAVE_PTHREAD_ATTR_GET_NP)) && defined(HAVE_PTHREAD_ATTR_GETSTACK)
4080   {
4081      size_t size;
4082      void *sstart;
4083      pthread_attr_t attr;
4084
4085 #if defined(HAVE_PTHREAD_GETATTR_NP)
4086     /* Linux */
4087     pthread_getattr_np (pthread_self (), &attr);
4088 #elif defined(HAVE_PTHREAD_ATTR_GET_NP)
4089     /* BSD */
4090     pthread_attr_init (&attr);
4091     pthread_attr_get_np (pthread_self (), &attr);
4092 #else
4093 #error Cannot determine which API is needed to retrieve pthread attributes.
4094 #endif
4095
4096      pthread_attr_getstack (&attr, &sstart, &size);
4097      info->stack_start_limit = sstart;
4098      info->stack_end = (char*)sstart + size;
4099      pthread_attr_destroy (&attr);
4100   }
4101 #elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
4102                  info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
4103                  info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
4104 #else
4105         {
4106                 /* FIXME: we assume the stack grows down */
4107                 gsize stack_bottom = (gsize)addr;
4108                 stack_bottom += 4095;
4109                 stack_bottom &= ~4095;
4110                 info->stack_end = (char*)stack_bottom;
4111         }
4112 #endif
4113
4114 #ifdef HAVE_KW_THREAD
4115         stack_end = info->stack_end;
4116 #endif
4117
4118         SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
4119
4120         if (gc_callbacks.thread_attach_func)
4121                 info->runtime_data = gc_callbacks.thread_attach_func ();
4122
4123         UNLOCK_GC;
4124         return info;
4125 }
4126
4127 static void
4128 sgen_thread_unregister (SgenThreadInfo *p)
4129 {
4130         /* If a delegate is passed to native code and invoked on a thread we dont
4131          * know about, the jit will register it with mono_jit_thread_attach, but
4132          * we have no way of knowing when that thread goes away.  SGen has a TSD
4133          * so we assume that if the domain is still registered, we can detach
4134          * the thread
4135          */
4136         if (mono_domain_get ())
4137                 mono_thread_detach (mono_thread_current ());
4138
4139         p->thread_is_dying = TRUE;
4140
4141         /*
4142         There is a race condition between a thread finishing executing and been removed
4143         from the GC thread set.
4144         This happens on posix systems when TLS data is been cleaned-up, libpthread will
4145         set the thread_info slot to NULL before calling the cleanup function. This
4146         opens a window in which the thread is registered but has a NULL TLS.
4147
4148         The suspend signal handler needs TLS data to know where to store thread state
4149         data or otherwise it will simply ignore the thread.
4150
4151         This solution works because the thread doing STW will wait until all threads been
4152         suspended handshake back, so there is no race between the doing_hankshake test
4153         and the suspend_thread call.
4154
4155         This is not required on systems that do synchronous STW as those can deal with
4156         the above race at suspend time.
4157
4158         FIXME: I believe we could avoid this by using mono_thread_info_lookup when
4159         mono_thread_info_current returns NULL. Or fix mono_thread_info_lookup to do so.
4160         */
4161 #if (defined(__MACH__) && MONO_MACH_ARCH_SUPPORTED) || !defined(HAVE_PTHREAD_KILL)
4162         LOCK_GC;
4163 #else
4164         while (!TRYLOCK_GC) {
4165                 if (!sgen_park_current_thread_if_doing_handshake (p))
4166                         g_usleep (50);
4167         }
4168         MONO_GC_LOCKED ();
4169 #endif
4170
4171         binary_protocol_thread_unregister ((gpointer)mono_thread_info_get_tid (p));
4172         SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)mono_thread_info_get_tid (p));
4173
4174         if (gc_callbacks.thread_detach_func) {
4175                 gc_callbacks.thread_detach_func (p->runtime_data);
4176                 p->runtime_data = NULL;
4177         }
4178
4179         mono_threads_unregister_current_thread (p);
4180         UNLOCK_GC;
4181 }
4182
4183
4184 static void
4185 sgen_thread_attach (SgenThreadInfo *info)
4186 {
4187         LOCK_GC;
4188         /*this is odd, can we get attached before the gc is inited?*/
4189         init_stats ();
4190         UNLOCK_GC;
4191
4192         if (gc_callbacks.thread_attach_func && !info->runtime_data)
4193                 info->runtime_data = gc_callbacks.thread_attach_func ();
4194 }
4195 gboolean
4196 mono_gc_register_thread (void *baseptr)
4197 {
4198         return mono_thread_info_attach (baseptr) != NULL;
4199 }
4200
4201 /*
4202  * mono_gc_set_stack_end:
4203  *
4204  *   Set the end of the current threads stack to STACK_END. The stack space between
4205  * STACK_END and the real end of the threads stack will not be scanned during collections.
4206  */
4207 void
4208 mono_gc_set_stack_end (void *stack_end)
4209 {
4210         SgenThreadInfo *info;
4211
4212         LOCK_GC;
4213         info = mono_thread_info_current ();
4214         if (info) {
4215                 g_assert (stack_end < info->stack_end);
4216                 info->stack_end = stack_end;
4217         }
4218         UNLOCK_GC;
4219 }
4220
4221 #if USE_PTHREAD_INTERCEPT
4222
4223
4224 int
4225 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
4226 {
4227         return pthread_create (new_thread, attr, start_routine, arg);
4228 }
4229
4230 int
4231 mono_gc_pthread_join (pthread_t thread, void **retval)
4232 {
4233         return pthread_join (thread, retval);
4234 }
4235
4236 int
4237 mono_gc_pthread_detach (pthread_t thread)
4238 {
4239         return pthread_detach (thread);
4240 }
4241
4242 void
4243 mono_gc_pthread_exit (void *retval)
4244 {
4245         mono_thread_info_dettach ();
4246         pthread_exit (retval);
4247 }
4248
4249 #endif /* USE_PTHREAD_INTERCEPT */
4250
4251 /*
4252  * ######################################################################
4253  * ########  Write barriers
4254  * ######################################################################
4255  */
4256
4257 /*
4258  * Note: the write barriers first do the needed GC work and then do the actual store:
4259  * this way the value is visible to the conservative GC scan after the write barrier
4260  * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
4261  * the conservative scan, otherwise by the remembered set scan.
4262  */
4263 void
4264 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
4265 {
4266         HEAVY_STAT (++stat_wbarrier_set_field);
4267         if (ptr_in_nursery (field_ptr)) {
4268                 *(void**)field_ptr = value;
4269                 return;
4270         }
4271         SGEN_LOG (8, "Adding remset at %p", field_ptr);
4272         if (value)
4273                 binary_protocol_wbarrier (field_ptr, value, value->vtable);
4274
4275         remset.wbarrier_set_field (obj, field_ptr, value);
4276 }
4277
4278 void
4279 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
4280 {
4281         HEAVY_STAT (++stat_wbarrier_set_arrayref);
4282         if (ptr_in_nursery (slot_ptr)) {
4283                 *(void**)slot_ptr = value;
4284                 return;
4285         }
4286         SGEN_LOG (8, "Adding remset at %p", slot_ptr);
4287         if (value)
4288                 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
4289
4290         remset.wbarrier_set_arrayref (arr, slot_ptr, value);
4291 }
4292
4293 void
4294 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
4295 {
4296         HEAVY_STAT (++stat_wbarrier_arrayref_copy);
4297         /*This check can be done without taking a lock since dest_ptr array is pinned*/
4298         if (ptr_in_nursery (dest_ptr) || count <= 0) {
4299                 mono_gc_memmove (dest_ptr, src_ptr, count * sizeof (gpointer));
4300                 return;
4301         }
4302
4303 #ifdef SGEN_BINARY_PROTOCOL
4304         {
4305                 int i;
4306                 for (i = 0; i < count; ++i) {
4307                         gpointer dest = (gpointer*)dest_ptr + i;
4308                         gpointer obj = *((gpointer*)src_ptr + i);
4309                         if (obj)
4310                                 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
4311                 }
4312         }
4313 #endif
4314
4315         remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
4316 }
4317
4318 static char *found_obj;
4319
4320 static void
4321 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4322 {
4323         char *ptr = user_data;
4324
4325         if (ptr >= obj && ptr < obj + size) {
4326                 g_assert (!found_obj);
4327                 found_obj = obj;
4328         }
4329 }
4330
4331 /* for use in the debugger */
4332 char* find_object_for_ptr (char *ptr);
4333 char*
4334 find_object_for_ptr (char *ptr)
4335 {
4336         if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4337                 found_obj = NULL;
4338                 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4339                                 find_object_for_ptr_callback, ptr, TRUE);
4340                 if (found_obj)
4341                         return found_obj;
4342         }
4343
4344         found_obj = NULL;
4345         sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4346         if (found_obj)
4347                 return found_obj;
4348
4349         /*
4350          * Very inefficient, but this is debugging code, supposed to
4351          * be called from gdb, so we don't care.
4352          */
4353         found_obj = NULL;
4354         major_collector.iterate_objects (TRUE, TRUE, find_object_for_ptr_callback, ptr);
4355         return found_obj;
4356 }
4357
4358 void
4359 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4360 {
4361         gpointer obj;
4362
4363         HEAVY_STAT (++stat_wbarrier_generic_store);
4364
4365 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4366         /* FIXME: ptr_in_heap must be called with the GC lock held */
4367         if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4368                 char *start = find_object_for_ptr (ptr);
4369                 MonoObject *value = *(MonoObject**)ptr;
4370                 LOCK_GC;
4371                 g_assert (start);
4372                 if (start) {
4373                         MonoObject *obj = (MonoObject*)start;
4374                         if (obj->vtable->domain != value->vtable->domain)
4375                                 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4376                 }
4377                 UNLOCK_GC;
4378         }
4379 #endif
4380
4381         obj = *(gpointer*)ptr;
4382         if (obj)
4383                 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4384
4385         if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4386                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4387                 return;
4388         }
4389
4390         /*
4391          * We need to record old->old pointer locations for the
4392          * concurrent collector.
4393          */
4394         if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4395                 SGEN_LOG (8, "Skipping remset at %p", ptr);
4396                 return;
4397         }
4398
4399         SGEN_LOG (8, "Adding remset at %p", ptr);
4400
4401         remset.wbarrier_generic_nostore (ptr);
4402 }
4403
4404 void
4405 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4406 {
4407         SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4408         *(void**)ptr = value;
4409         if (ptr_in_nursery (value))
4410                 mono_gc_wbarrier_generic_nostore (ptr);
4411         sgen_dummy_use (value);
4412 }
4413
4414 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4415 {
4416         mword *dest = _dest;
4417         mword *src = _src;
4418
4419         while (size) {
4420                 if (bitmap & 0x1)
4421                         mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4422                 else
4423                         *dest = *src;
4424                 ++src;
4425                 ++dest;
4426                 size -= SIZEOF_VOID_P;
4427                 bitmap >>= 1;
4428         }
4429 }
4430
4431 #ifdef SGEN_BINARY_PROTOCOL
4432 #undef HANDLE_PTR
4433 #define HANDLE_PTR(ptr,obj) do {                                        \
4434                 gpointer o = *(gpointer*)(ptr);                         \
4435                 if ((o)) {                                              \
4436                         gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4437                         binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4438                 }                                                       \
4439         } while (0)
4440
4441 static void
4442 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4443 {
4444 #define SCAN_OBJECT_NOVTABLE
4445 #include "sgen-scan-object.h"
4446 }
4447 #endif
4448
4449 void
4450 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4451 {
4452         HEAVY_STAT (++stat_wbarrier_value_copy);
4453         g_assert (klass->valuetype);
4454
4455         SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4456
4457         if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4458                 size_t element_size = mono_class_value_size (klass, NULL);
4459                 size_t size = count * element_size;
4460                 mono_gc_memmove (dest, src, size);
4461                 return;
4462         }
4463
4464 #ifdef SGEN_BINARY_PROTOCOL
4465         {
4466                 size_t element_size = mono_class_value_size (klass, NULL);
4467                 int i;
4468                 for (i = 0; i < count; ++i) {
4469                         scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4470                                         (char*)src + i * element_size - sizeof (MonoObject),
4471                                         (mword) klass->gc_descr);
4472                 }
4473         }
4474 #endif
4475
4476         remset.wbarrier_value_copy (dest, src, count, klass);
4477 }
4478
4479 /**
4480  * mono_gc_wbarrier_object_copy:
4481  *
4482  * Write barrier to call when obj is the result of a clone or copy of an object.
4483  */
4484 void
4485 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4486 {
4487         int size;
4488
4489         HEAVY_STAT (++stat_wbarrier_object_copy);
4490
4491         if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4492                 size = mono_object_class (obj)->instance_size;
4493                 mono_gc_memmove ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4494                                 size - sizeof (MonoObject));
4495                 return;
4496         }
4497
4498 #ifdef SGEN_BINARY_PROTOCOL
4499         scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4500 #endif
4501
4502         remset.wbarrier_object_copy (obj, src);
4503 }
4504
4505
4506 /*
4507  * ######################################################################
4508  * ########  Other mono public interface functions.
4509  * ######################################################################
4510  */
4511
4512 #define REFS_SIZE 128
4513 typedef struct {
4514         void *data;
4515         MonoGCReferences callback;
4516         int flags;
4517         int count;
4518         int called;
4519         MonoObject *refs [REFS_SIZE];
4520         uintptr_t offsets [REFS_SIZE];
4521 } HeapWalkInfo;
4522
4523 #undef HANDLE_PTR
4524 #define HANDLE_PTR(ptr,obj)     do {    \
4525                 if (*(ptr)) {   \
4526                         if (hwi->count == REFS_SIZE) {  \
4527                                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);    \
4528                                 hwi->count = 0; \
4529                                 hwi->called = 1;        \
4530                         }       \
4531                         hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start;  \
4532                         hwi->refs [hwi->count++] = *(ptr);      \
4533                 }       \
4534         } while (0)
4535
4536 static void
4537 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4538 {
4539 #include "sgen-scan-object.h"
4540 }
4541
4542 static void
4543 walk_references (char *start, size_t size, void *data)
4544 {
4545         HeapWalkInfo *hwi = data;
4546         hwi->called = 0;
4547         hwi->count = 0;
4548         collect_references (hwi, start, size);
4549         if (hwi->count || !hwi->called)
4550                 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4551 }
4552
4553 /**
4554  * mono_gc_walk_heap:
4555  * @flags: flags for future use
4556  * @callback: a function pointer called for each object in the heap
4557  * @data: a user data pointer that is passed to callback
4558  *
4559  * This function can be used to iterate over all the live objects in the heap:
4560  * for each object, @callback is invoked, providing info about the object's
4561  * location in memory, its class, its size and the objects it references.
4562  * For each referenced object it's offset from the object address is
4563  * reported in the offsets array.
4564  * The object references may be buffered, so the callback may be invoked
4565  * multiple times for the same object: in all but the first call, the size
4566  * argument will be zero.
4567  * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4568  * profiler event handler.
4569  *
4570  * Returns: a non-zero value if the GC doesn't support heap walking
4571  */
4572 int
4573 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4574 {
4575         HeapWalkInfo hwi;
4576
4577         hwi.flags = flags;
4578         hwi.callback = callback;
4579         hwi.data = data;
4580
4581         sgen_clear_nursery_fragments ();
4582         sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4583
4584         major_collector.iterate_objects (TRUE, TRUE, walk_references, &hwi);
4585         sgen_los_iterate_objects (walk_references, &hwi);
4586
4587         return 0;
4588 }
4589
4590 void
4591 mono_gc_collect (int generation)
4592 {
4593         LOCK_GC;
4594         if (generation > 1)
4595                 generation = 1;
4596         sgen_perform_collection (0, generation, "user request", TRUE);
4597         UNLOCK_GC;
4598 }
4599
4600 int
4601 mono_gc_max_generation (void)
4602 {
4603         return 1;
4604 }
4605
4606 int
4607 mono_gc_collection_count (int generation)
4608 {
4609         if (generation == 0)
4610                 return stat_minor_gcs;
4611         return stat_major_gcs;
4612 }
4613
4614 int64_t
4615 mono_gc_get_used_size (void)
4616 {
4617         gint64 tot = 0;
4618         LOCK_GC;
4619         tot = los_memory_usage;
4620         tot += nursery_section->next_data - nursery_section->data;
4621         tot += major_collector.get_used_size ();
4622         /* FIXME: account for pinned objects */
4623         UNLOCK_GC;
4624         return tot;
4625 }
4626
4627 int
4628 mono_gc_get_los_limit (void)
4629 {
4630         return MAX_SMALL_OBJ_SIZE;
4631 }
4632
4633 gboolean
4634 mono_gc_user_markers_supported (void)
4635 {
4636         return TRUE;
4637 }
4638
4639 gboolean
4640 mono_object_is_alive (MonoObject* o)
4641 {
4642         return TRUE;
4643 }
4644
4645 int
4646 mono_gc_get_generation (MonoObject *obj)
4647 {
4648         if (ptr_in_nursery (obj))
4649                 return 0;
4650         return 1;
4651 }
4652
4653 void
4654 mono_gc_enable_events (void)
4655 {
4656 }
4657
4658 void
4659 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4660 {
4661         sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4662 }
4663
4664 void
4665 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4666 {
4667         sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4668 }
4669
4670 MonoObject*
4671 mono_gc_weak_link_get (void **link_addr)
4672 {
4673         void * volatile *link_addr_volatile;
4674         void *ptr;
4675         MonoObject *obj;
4676  retry:
4677         link_addr_volatile = link_addr;
4678         ptr = (void*)*link_addr_volatile;
4679         /*
4680          * At this point we have a hidden pointer.  If the GC runs
4681          * here, it will not recognize the hidden pointer as a
4682          * reference, and if the object behind it is not referenced
4683          * elsewhere, it will be freed.  Once the world is restarted
4684          * we reveal the pointer, giving us a pointer to a freed
4685          * object.  To make sure we don't return it, we load the
4686          * hidden pointer again.  If it's still the same, we can be
4687          * sure the object reference is valid.
4688          */
4689         if (ptr)
4690                 obj = (MonoObject*) REVEAL_POINTER (ptr);
4691         else
4692                 return NULL;
4693
4694         mono_memory_barrier ();
4695
4696         /*
4697          * During the second bridge processing step the world is
4698          * running again.  That step processes all weak links once
4699          * more to null those that refer to dead objects.  Before that
4700          * is completed, those links must not be followed, so we
4701          * conservatively wait for bridge processing when any weak
4702          * link is dereferenced.
4703          */
4704         if (G_UNLIKELY (bridge_processing_in_progress))
4705                 mono_gc_wait_for_bridge_processing ();
4706
4707         if ((void*)*link_addr_volatile != ptr)
4708                 goto retry;
4709
4710         return obj;
4711 }
4712
4713 gboolean
4714 mono_gc_ephemeron_array_add (MonoObject *obj)
4715 {
4716         EphemeronLinkNode *node;
4717
4718         LOCK_GC;
4719
4720         node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4721         if (!node) {
4722                 UNLOCK_GC;
4723                 return FALSE;
4724         }
4725         node->array = (char*)obj;
4726         node->next = ephemeron_list;
4727         ephemeron_list = node;
4728
4729         SGEN_LOG (5, "Registered ephemeron array %p", obj);
4730
4731         UNLOCK_GC;
4732         return TRUE;
4733 }
4734
4735 gboolean
4736 mono_gc_set_allow_synchronous_major (gboolean flag)
4737 {
4738         if (!major_collector.is_concurrent)
4739                 return flag;
4740
4741         allow_synchronous_major = flag;
4742         return TRUE;
4743 }
4744
4745 void*
4746 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4747 {
4748         void *result;
4749         LOCK_INTERRUPTION;
4750         result = func (data);
4751         UNLOCK_INTERRUPTION;
4752         return result;
4753 }
4754
4755 gboolean
4756 mono_gc_is_gc_thread (void)
4757 {
4758         gboolean result;
4759         LOCK_GC;
4760         result = mono_thread_info_current () != NULL;
4761         UNLOCK_GC;
4762         return result;
4763 }
4764
4765 static gboolean
4766 is_critical_method (MonoMethod *method)
4767 {
4768         return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4769 }
4770
4771 void
4772 sgen_env_var_error (const char *env_var, const char *fallback, const char *description_format, ...)
4773 {
4774         va_list ap;
4775
4776         va_start (ap, description_format);
4777
4778         fprintf (stderr, "Warning: In environment variable `%s': ", env_var);
4779         vfprintf (stderr, description_format, ap);
4780         if (fallback)
4781                 fprintf (stderr, " - %s", fallback);
4782         fprintf (stderr, "\n");
4783
4784         va_end (ap);
4785 }
4786
4787 static gboolean
4788 parse_double_in_interval (const char *env_var, const char *opt_name, const char *opt, double min, double max, double *result)
4789 {
4790         char *endptr;
4791         double val = strtod (opt, &endptr);
4792         if (endptr == opt) {
4793                 sgen_env_var_error (env_var, "Using default value.", "`%s` must be a number.", opt_name);
4794                 return FALSE;
4795         }
4796         else if (val < min || val > max) {
4797                 sgen_env_var_error (env_var, "Using default value.", "`%s` must be between %.2f - %.2f.", opt_name, min, max);
4798                 return FALSE;
4799         }
4800         *result = val;
4801         return TRUE;
4802 }
4803
4804 void
4805 mono_gc_base_init (void)
4806 {
4807         MonoThreadInfoCallbacks cb;
4808         char *env;
4809         char **opts, **ptr;
4810         char *major_collector_opt = NULL;
4811         char *minor_collector_opt = NULL;
4812         glong max_heap = 0;
4813         glong soft_limit = 0;
4814         int num_workers;
4815         int result;
4816         int dummy;
4817         gboolean debug_print_allowance = FALSE;
4818         double allowance_ratio = 0, save_target = 0;
4819         gboolean have_split_nursery = FALSE;
4820         gboolean cement_enabled = TRUE;
4821
4822         do {
4823                 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4824                 switch (result) {
4825                 case 1:
4826                         /* already inited */
4827                         return;
4828                 case -1:
4829                         /* being inited by another thread */
4830                         g_usleep (1000);
4831                         break;
4832                 case 0:
4833                         /* we will init it */
4834                         break;
4835                 default:
4836                         g_assert_not_reached ();
4837                 }
4838         } while (result != 0);
4839
4840         LOCK_INIT (gc_mutex);
4841
4842         pagesize = mono_pagesize ();
4843         gc_debug_file = stderr;
4844
4845         cb.thread_register = sgen_thread_register;
4846         cb.thread_unregister = sgen_thread_unregister;
4847         cb.thread_attach = sgen_thread_attach;
4848         cb.mono_method_is_critical = (gpointer)is_critical_method;
4849 #ifndef HOST_WIN32
4850         cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4851 #endif
4852
4853         mono_threads_init (&cb, sizeof (SgenThreadInfo));
4854
4855         LOCK_INIT (sgen_interruption_mutex);
4856         LOCK_INIT (pin_queue_mutex);
4857
4858         init_user_copy_or_mark_key ();
4859
4860         if ((env = getenv (MONO_GC_PARAMS_NAME))) {
4861                 opts = g_strsplit (env, ",", -1);
4862                 for (ptr = opts; *ptr; ++ptr) {
4863                         char *opt = *ptr;
4864                         if (g_str_has_prefix (opt, "major=")) {
4865                                 opt = strchr (opt, '=') + 1;
4866                                 major_collector_opt = g_strdup (opt);
4867                         } else if (g_str_has_prefix (opt, "minor=")) {
4868                                 opt = strchr (opt, '=') + 1;
4869                                 minor_collector_opt = g_strdup (opt);
4870                         }
4871                 }
4872         } else {
4873                 opts = NULL;
4874         }
4875
4876         init_stats ();
4877         sgen_init_internal_allocator ();
4878         sgen_init_nursery_allocator ();
4879         sgen_init_fin_weak_hash ();
4880
4881         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4882         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4883         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4884         sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4885
4886 #ifndef HAVE_KW_THREAD
4887         mono_native_tls_alloc (&thread_info_key, NULL);
4888 #endif
4889
4890         /*
4891          * This needs to happen before any internal allocations because
4892          * it inits the small id which is required for hazard pointer
4893          * operations.
4894          */
4895         sgen_os_init ();
4896
4897         mono_thread_info_attach (&dummy);
4898
4899         if (!minor_collector_opt) {
4900                 sgen_simple_nursery_init (&sgen_minor_collector);
4901         } else {
4902                 if (!strcmp (minor_collector_opt, "simple")) {
4903                 use_simple_nursery:
4904                         sgen_simple_nursery_init (&sgen_minor_collector);
4905                 } else if (!strcmp (minor_collector_opt, "split")) {
4906                         sgen_split_nursery_init (&sgen_minor_collector);
4907                         have_split_nursery = TRUE;
4908                 } else {
4909                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `simple` instead.", "Unknown minor collector `%s'.", minor_collector_opt);
4910                         goto use_simple_nursery;
4911                 }
4912         }
4913
4914         if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4915         use_marksweep_major:
4916                 sgen_marksweep_init (&major_collector);
4917         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed")) {
4918                 sgen_marksweep_fixed_init (&major_collector);
4919         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-par")) {
4920                 sgen_marksweep_par_init (&major_collector);
4921         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) {
4922                 sgen_marksweep_fixed_par_init (&major_collector);
4923         } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4924                 sgen_marksweep_conc_init (&major_collector);
4925         } else {
4926                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `marksweep` instead.", "Unknown major collector `%s'.", major_collector_opt);
4927                 goto use_marksweep_major;
4928         }
4929
4930         if (have_split_nursery && major_collector.is_parallel) {
4931                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Disabling split minor collector.", "`minor=split` is not supported with the parallel collector yet.");
4932                 have_split_nursery = FALSE;
4933         }
4934
4935         num_workers = mono_cpu_count ();
4936         g_assert (num_workers > 0);
4937         if (num_workers > 16)
4938                 num_workers = 16;
4939
4940         ///* Keep this the default for now */
4941         /* Precise marking is broken on all supported targets. Disable until fixed. */
4942         conservative_stack_mark = TRUE;
4943
4944         sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4945
4946         if (opts) {
4947                 gboolean usage_printed = FALSE;
4948
4949                 for (ptr = opts; *ptr; ++ptr) {
4950                         char *opt = *ptr;
4951                         if (!strcmp (opt, ""))
4952                                 continue;
4953                         if (g_str_has_prefix (opt, "major="))
4954                                 continue;
4955                         if (g_str_has_prefix (opt, "minor="))
4956                                 continue;
4957                         if (g_str_has_prefix (opt, "max-heap-size=")) {
4958                                 glong max_heap_candidate = 0;
4959                                 opt = strchr (opt, '=') + 1;
4960                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap_candidate)) {
4961                                         max_heap = (max_heap_candidate + mono_pagesize () - 1) & ~(glong)(mono_pagesize () - 1);
4962                                         if (max_heap != max_heap_candidate)
4963                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Rounding up.", "`max-heap-size` size must be a multiple of %d.", mono_pagesize ());
4964                                 } else {
4965                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`max-heap-size` must be an integer.");
4966                                 }
4967                                 continue;
4968                         }
4969                         if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4970                                 opt = strchr (opt, '=') + 1;
4971                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4972                                         if (soft_limit <= 0) {
4973                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be positive.");
4974                                                 soft_limit = 0;
4975                                         }
4976                                 } else {
4977                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be an integer.");
4978                                 }
4979                                 continue;
4980                         }
4981                         if (g_str_has_prefix (opt, "workers=")) {
4982                                 long val;
4983                                 char *endptr;
4984                                 if (!major_collector.is_parallel) {
4985                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "The `workers` option can only be used for parallel collectors.");
4986                                         continue;
4987                                 }
4988                                 opt = strchr (opt, '=') + 1;
4989                                 val = strtol (opt, &endptr, 10);
4990                                 if (!*opt || *endptr) {
4991                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Cannot parse the `workers` option value.");
4992                                         continue;
4993                                 }
4994                                 if (val <= 0 || val > 16) {
4995                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "The number of `workers` must be in the range 1 to 16.");
4996                                         continue;
4997                                 }
4998                                 num_workers = (int)val;
4999                                 continue;
5000                         }
5001                         if (g_str_has_prefix (opt, "stack-mark=")) {
5002                                 opt = strchr (opt, '=') + 1;
5003                                 if (!strcmp (opt, "precise")) {
5004                                         conservative_stack_mark = FALSE;
5005                                 } else if (!strcmp (opt, "conservative")) {
5006                                         conservative_stack_mark = TRUE;
5007                                 } else {
5008                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, conservative_stack_mark ? "Using `conservative`." : "Using `precise`.",
5009                                                         "Invalid value `%s` for `stack-mark` option, possible values are: `precise`, `conservative`.", opt);
5010                                 }
5011                                 continue;
5012                         }
5013                         if (g_str_has_prefix (opt, "bridge=")) {
5014                                 opt = strchr (opt, '=') + 1;
5015                                 sgen_register_test_bridge_callbacks (g_strdup (opt));
5016                                 continue;
5017                         }
5018 #ifdef USER_CONFIG
5019                         if (g_str_has_prefix (opt, "nursery-size=")) {
5020                                 long val;
5021                                 opt = strchr (opt, '=') + 1;
5022                                 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
5023 #ifdef SGEN_ALIGN_NURSERY
5024                                         if ((val & (val - 1))) {
5025                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be a power of two.");
5026                                                 continue;
5027                                         }
5028
5029                                         if (val < SGEN_MAX_NURSERY_WASTE) {
5030                                                 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.",
5031                                                                 "`nursery-size` must be at least %d bytes.\n", SGEN_MAX_NURSERY_WASTE);
5032                                                 continue;
5033                                         }
5034
5035                                         sgen_nursery_size = val;
5036                                         sgen_nursery_bits = 0;
5037                                         while (1 << (++ sgen_nursery_bits) != sgen_nursery_size)
5038                                                 ;
5039 #else
5040                                         sgen_nursery_size = val;
5041 #endif
5042                                 } else {
5043                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be an integer.");
5044                                         continue;
5045                                 }
5046                                 continue;
5047                         }
5048 #endif
5049                         if (g_str_has_prefix (opt, "save-target-ratio=")) {
5050                                 double val;
5051                                 opt = strchr (opt, '=') + 1;
5052                                 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "save-target-ratio", opt,
5053                                                 SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO, &val)) {
5054                                         save_target = val;
5055                                 }
5056                                 continue;
5057                         }
5058                         if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
5059                                 double val;
5060                                 opt = strchr (opt, '=') + 1;
5061                                 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "default-allowance-ratio", opt,
5062                                                 SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, &val)) {
5063                                         allowance_ratio = val;
5064                                 }
5065                                 continue;
5066                         }
5067                         if (g_str_has_prefix (opt, "allow-synchronous-major=")) {
5068                                 if (!major_collector.is_concurrent) {
5069                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "`allow-synchronous-major` is only valid for the concurrent major collector.");
5070                                         continue;
5071                                 }
5072
5073                                 opt = strchr (opt, '=') + 1;
5074
5075                                 if (!strcmp (opt, "yes")) {
5076                                         allow_synchronous_major = TRUE;
5077                                 } else if (!strcmp (opt, "no")) {
5078                                         allow_synchronous_major = FALSE;
5079                                 } else {
5080                                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`allow-synchronous-major` must be either `yes' or `no'.");
5081                                         continue;
5082                                 }
5083                         }
5084
5085                         if (!strcmp (opt, "cementing")) {
5086                                 cement_enabled = TRUE;
5087                                 continue;
5088                         }
5089                         if (!strcmp (opt, "no-cementing")) {
5090                                 cement_enabled = FALSE;
5091                                 continue;
5092                         }
5093
5094                         if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
5095                                 continue;
5096
5097                         if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
5098                                 continue;
5099
5100                         sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Unknown option `%s`.", opt);
5101
5102                         if (usage_printed)
5103                                 continue;
5104
5105                         fprintf (stderr, "\n%s must be a comma-delimited list of one or more of the following:\n", MONO_GC_PARAMS_NAME);
5106                         fprintf (stderr, "  max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5107                         fprintf (stderr, "  soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
5108                         fprintf (stderr, "  nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
5109                         fprintf (stderr, "  major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-conc', `marksweep-par', 'marksweep-fixed' or 'marksweep-fixed-par')\n");
5110                         fprintf (stderr, "  minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
5111                         fprintf (stderr, "  wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
5112                         fprintf (stderr, "  stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
5113                         fprintf (stderr, "  [no-]cementing\n");
5114                         if (major_collector.is_concurrent)
5115                                 fprintf (stderr, "  allow-synchronous-major=FLAG (where FLAG is `yes' or `no')\n");
5116                         if (major_collector.print_gc_param_usage)
5117                                 major_collector.print_gc_param_usage ();
5118                         if (sgen_minor_collector.print_gc_param_usage)
5119                                 sgen_minor_collector.print_gc_param_usage ();
5120                         fprintf (stderr, " Experimental options:\n");
5121                         fprintf (stderr, "  save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
5122                         fprintf (stderr, "  default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
5123                         fprintf (stderr, "\n");
5124
5125                         usage_printed = TRUE;
5126                 }
5127                 g_strfreev (opts);
5128         }
5129
5130         if (major_collector.is_parallel)
5131                 sgen_workers_init (num_workers);
5132         else if (major_collector.is_concurrent)
5133                 sgen_workers_init (1);
5134
5135         if (major_collector_opt)
5136                 g_free (major_collector_opt);
5137
5138         if (minor_collector_opt)
5139                 g_free (minor_collector_opt);
5140
5141         alloc_nursery ();
5142
5143         sgen_cement_init (cement_enabled);
5144
5145         if ((env = getenv (MONO_GC_DEBUG_NAME))) {
5146                 gboolean usage_printed = FALSE;
5147
5148                 opts = g_strsplit (env, ",", -1);
5149                 for (ptr = opts; ptr && *ptr; ptr ++) {
5150                         char *opt = *ptr;
5151                         if (!strcmp (opt, ""))
5152                                 continue;
5153                         if (opt [0] >= '0' && opt [0] <= '9') {
5154                                 gc_debug_level = atoi (opt);
5155                                 opt++;
5156                                 if (opt [0] == ':')
5157                                         opt++;
5158                                 if (opt [0]) {
5159 #ifdef HOST_WIN32
5160                                         char *rf = g_strdup_printf ("%s.%d", opt, GetCurrentProcessId ());
5161 #else
5162                                         char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
5163 #endif
5164                                         gc_debug_file = fopen (rf, "wb");
5165                                         if (!gc_debug_file)
5166                                                 gc_debug_file = stderr;
5167                                         g_free (rf);
5168                                 }
5169                         } else if (!strcmp (opt, "print-allowance")) {
5170                                 debug_print_allowance = TRUE;
5171                         } else if (!strcmp (opt, "print-pinning")) {
5172                                 do_pin_stats = TRUE;
5173                         } else if (!strcmp (opt, "verify-before-allocs")) {
5174                                 verify_before_allocs = 1;
5175                                 has_per_allocation_action = TRUE;
5176                         } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
5177                                 char *arg = strchr (opt, '=') + 1;
5178                                 verify_before_allocs = atoi (arg);
5179                                 has_per_allocation_action = TRUE;
5180                         } else if (!strcmp (opt, "collect-before-allocs")) {
5181                                 collect_before_allocs = 1;
5182                                 has_per_allocation_action = TRUE;
5183                         } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
5184                                 char *arg = strchr (opt, '=') + 1;
5185                                 has_per_allocation_action = TRUE;
5186                                 collect_before_allocs = atoi (arg);
5187                         } else if (!strcmp (opt, "verify-before-collections")) {
5188                                 whole_heap_check_before_collection = TRUE;
5189                         } else if (!strcmp (opt, "check-at-minor-collections")) {
5190                                 consistency_check_at_minor_collection = TRUE;
5191                                 nursery_clear_policy = CLEAR_AT_GC;
5192                         } else if (!strcmp (opt, "mod-union-consistency-check")) {
5193                                 if (!major_collector.is_concurrent) {
5194                                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`mod-union-consistency-check` only works with concurrent major collector.");
5195                                         continue;
5196                                 }
5197                                 mod_union_consistency_check = TRUE;
5198                         } else if (!strcmp (opt, "check-mark-bits")) {
5199                                 check_mark_bits_after_major_collection = TRUE;
5200                         } else if (!strcmp (opt, "check-nursery-pinned")) {
5201                                 check_nursery_objects_pinned = TRUE;
5202                         } else if (!strcmp (opt, "xdomain-checks")) {
5203                                 xdomain_checks = TRUE;
5204                         } else if (!strcmp (opt, "clear-at-gc")) {
5205                                 nursery_clear_policy = CLEAR_AT_GC;
5206                         } else if (!strcmp (opt, "clear-nursery-at-gc")) {
5207                                 nursery_clear_policy = CLEAR_AT_GC;
5208                         } else if (!strcmp (opt, "check-scan-starts")) {
5209                                 do_scan_starts_check = TRUE;
5210                         } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
5211                                 do_verify_nursery = TRUE;
5212                         } else if (!strcmp (opt, "check-concurrent")) {
5213                                 if (!major_collector.is_concurrent) {
5214                                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`check-concurrent` only works with concurrent major collectors.");
5215                                         continue;
5216                                 }
5217                                 do_concurrent_checks = TRUE;
5218                         } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
5219                                 do_dump_nursery_content = TRUE;
5220                         } else if (!strcmp (opt, "no-managed-allocator")) {
5221                                 sgen_set_use_managed_allocator (FALSE);
5222                         } else if (!strcmp (opt, "disable-minor")) {
5223                                 disable_minor_collections = TRUE;
5224                         } else if (!strcmp (opt, "disable-major")) {
5225                                 disable_major_collections = TRUE;
5226                         } else if (g_str_has_prefix (opt, "heap-dump=")) {
5227                                 char *filename = strchr (opt, '=') + 1;
5228                                 nursery_clear_policy = CLEAR_AT_GC;
5229                                 heap_dump_file = fopen (filename, "w");
5230                                 if (heap_dump_file) {
5231                                         fprintf (heap_dump_file, "<sgen-dump>\n");
5232                                         do_pin_stats = TRUE;
5233                                 }
5234 #ifdef SGEN_BINARY_PROTOCOL
5235                         } else if (g_str_has_prefix (opt, "binary-protocol=")) {
5236                                 char *filename = strchr (opt, '=') + 1;
5237                                 binary_protocol_init (filename);
5238 #endif
5239                         } else {
5240                                 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "Unknown option `%s`.", opt);
5241
5242                                 if (usage_printed)
5243                                         continue;
5244
5245                                 fprintf (stderr, "\n%s must be of the format [<l>[:<filename>]|<option>]+ where <l> is a debug level 0-9.\n", MONO_GC_DEBUG_NAME);
5246                                 fprintf (stderr, "Valid <option>s are:\n");
5247                                 fprintf (stderr, "  collect-before-allocs[=<n>]\n");
5248                                 fprintf (stderr, "  verify-before-allocs[=<n>]\n");
5249                                 fprintf (stderr, "  check-at-minor-collections\n");
5250                                 fprintf (stderr, "  check-mark-bits\n");
5251                                 fprintf (stderr, "  check-nursery-pinned\n");
5252                                 fprintf (stderr, "  verify-before-collections\n");
5253                                 fprintf (stderr, "  verify-nursery-at-minor-gc\n");
5254                                 fprintf (stderr, "  dump-nursery-at-minor-gc\n");
5255                                 fprintf (stderr, "  disable-minor\n");
5256                                 fprintf (stderr, "  disable-major\n");
5257                                 fprintf (stderr, "  xdomain-checks\n");
5258                                 fprintf (stderr, "  check-concurrent\n");
5259                                 fprintf (stderr, "  clear-at-gc\n");
5260                                 fprintf (stderr, "  clear-nursery-at-gc\n");
5261                                 fprintf (stderr, "  check-scan-starts\n");
5262                                 fprintf (stderr, "  no-managed-allocator\n");
5263                                 fprintf (stderr, "  print-allowance\n");
5264                                 fprintf (stderr, "  print-pinning\n");
5265                                 fprintf (stderr, "  heap-dump=<filename>\n");
5266 #ifdef SGEN_BINARY_PROTOCOL
5267                                 fprintf (stderr, "  binary-protocol=<filename>\n");
5268 #endif
5269                                 fprintf (stderr, "\n");
5270
5271                                 usage_printed = TRUE;
5272                         }
5273                 }
5274                 g_strfreev (opts);
5275         }
5276
5277         if (major_collector.is_parallel) {
5278                 if (heap_dump_file) {
5279                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Disabling.", "Cannot do `heap-dump` with the parallel collector.");
5280                         fclose (heap_dump_file);
5281                         heap_dump_file = NULL;
5282                 }
5283                 if (do_pin_stats) {
5284                         sgen_env_var_error (MONO_GC_DEBUG_NAME, "Disabling.", "`print-pinning` is not supported with the parallel collector.");
5285                         do_pin_stats = FALSE;
5286                 }
5287         }
5288
5289         if (major_collector.post_param_init)
5290                 major_collector.post_param_init (&major_collector);
5291
5292         sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
5293
5294         memset (&remset, 0, sizeof (remset));
5295
5296         sgen_card_table_init (&remset);
5297
5298         gc_initialized = 1;
5299 }
5300
5301 const char *
5302 mono_gc_get_gc_name (void)
5303 {
5304         return "sgen";
5305 }
5306
5307 static MonoMethod *write_barrier_method;
5308
5309 gboolean
5310 sgen_is_critical_method (MonoMethod *method)
5311 {
5312         return (method == write_barrier_method || sgen_is_managed_allocator (method));
5313 }
5314
5315 gboolean
5316 sgen_has_critical_method (void)
5317 {
5318         return write_barrier_method || sgen_has_managed_allocator ();
5319 }
5320
5321 #ifndef DISABLE_JIT
5322
5323 static void
5324 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5325 {
5326         memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5327 #ifdef SGEN_ALIGN_NURSERY
5328         // if (ptr_in_nursery (ptr)) return;
5329         /*
5330          * Masking out the bits might be faster, but we would have to use 64 bit
5331          * immediates, which might be slower.
5332          */
5333         mono_mb_emit_ldarg (mb, 0);
5334         mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5335         mono_mb_emit_byte (mb, CEE_SHR_UN);
5336         mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5337         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5338
5339         if (!major_collector.is_concurrent) {
5340                 // if (!ptr_in_nursery (*ptr)) return;
5341                 mono_mb_emit_ldarg (mb, 0);
5342                 mono_mb_emit_byte (mb, CEE_LDIND_I);
5343                 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5344                 mono_mb_emit_byte (mb, CEE_SHR_UN);
5345                 mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS);
5346                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5347         }
5348 #else
5349         int label_continue1, label_continue2;
5350         int dereferenced_var;
5351
5352         // if (ptr < (sgen_get_nursery_start ())) goto continue;
5353         mono_mb_emit_ldarg (mb, 0);
5354         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5355         label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5356
5357         // if (ptr >= sgen_get_nursery_end ())) goto continue;
5358         mono_mb_emit_ldarg (mb, 0);
5359         mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5360         label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5361
5362         // Otherwise return
5363         nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5364
5365         // continue:
5366         mono_mb_patch_branch (mb, label_continue_1);
5367         mono_mb_patch_branch (mb, label_continue_2);
5368
5369         // Dereference and store in local var
5370         dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5371         mono_mb_emit_ldarg (mb, 0);
5372         mono_mb_emit_byte (mb, CEE_LDIND_I);
5373         mono_mb_emit_stloc (mb, dereferenced_var);
5374
5375         if (!major_collector.is_concurrent) {
5376                 // if (*ptr < sgen_get_nursery_start ()) return;
5377                 mono_mb_emit_ldloc (mb, dereferenced_var);
5378                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5379                 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5380
5381                 // if (*ptr >= sgen_get_nursery_end ()) return;
5382                 mono_mb_emit_ldloc (mb, dereferenced_var);
5383                 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5384                 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5385         }
5386 #endif
5387 }
5388 #endif
5389
5390 MonoMethod*
5391 mono_gc_get_write_barrier (void)
5392 {
5393         MonoMethod *res;
5394         MonoMethodBuilder *mb;
5395         MonoMethodSignature *sig;
5396 #ifdef MANAGED_WBARRIER
5397         int i, nursery_check_labels [3];
5398
5399 #ifdef HAVE_KW_THREAD
5400         int stack_end_offset = -1;
5401
5402         MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5403         g_assert (stack_end_offset != -1);
5404 #endif
5405 #endif
5406
5407         // FIXME: Maybe create a separate version for ctors (the branch would be
5408         // correctly predicted more times)
5409         if (write_barrier_method)
5410                 return write_barrier_method;
5411
5412         /* Create the IL version of mono_gc_barrier_generic_store () */
5413         sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5414         sig->ret = &mono_defaults.void_class->byval_arg;
5415         sig->params [0] = &mono_defaults.int_class->byval_arg;
5416
5417         mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5418
5419 #ifndef DISABLE_JIT
5420 #ifdef MANAGED_WBARRIER
5421         emit_nursery_check (mb, nursery_check_labels);
5422         /*
5423         addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5424         *addr = 1;
5425
5426         sgen_cardtable:
5427                 LDC_PTR sgen_cardtable
5428
5429         address >> CARD_BITS
5430                 LDARG_0
5431                 LDC_I4 CARD_BITS
5432                 SHR_UN
5433         if (SGEN_HAVE_OVERLAPPING_CARDS) {
5434                 LDC_PTR card_table_mask
5435                 AND
5436         }
5437         AND
5438         ldc_i4_1
5439         stind_i1
5440         */
5441         mono_mb_emit_ptr (mb, sgen_cardtable);
5442         mono_mb_emit_ldarg (mb, 0);
5443         mono_mb_emit_icon (mb, CARD_BITS);
5444         mono_mb_emit_byte (mb, CEE_SHR_UN);
5445 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5446         mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5447         mono_mb_emit_byte (mb, CEE_AND);
5448 #endif
5449         mono_mb_emit_byte (mb, CEE_ADD);
5450         mono_mb_emit_icon (mb, 1);
5451         mono_mb_emit_byte (mb, CEE_STIND_I1);
5452
5453         // return;
5454         for (i = 0; i < 3; ++i) {
5455                 if (nursery_check_labels [i])
5456                         mono_mb_patch_branch (mb, nursery_check_labels [i]);
5457         }
5458         mono_mb_emit_byte (mb, CEE_RET);
5459 #else
5460         mono_mb_emit_ldarg (mb, 0);
5461         mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5462         mono_mb_emit_byte (mb, CEE_RET);
5463 #endif
5464 #endif
5465         res = mono_mb_create_method (mb, sig, 16);
5466         mono_mb_free (mb);
5467
5468         mono_loader_lock ();
5469         if (write_barrier_method) {
5470                 /* Already created */
5471                 mono_free_method (res);
5472         } else {
5473                 /* double-checked locking */
5474                 mono_memory_barrier ();
5475                 write_barrier_method = res;
5476         }
5477         mono_loader_unlock ();
5478
5479         return write_barrier_method;
5480 }
5481
5482 char*
5483 mono_gc_get_description (void)
5484 {
5485         return g_strdup ("sgen");
5486 }
5487
5488 void
5489 mono_gc_set_desktop_mode (void)
5490 {
5491 }
5492
5493 gboolean
5494 mono_gc_is_moving (void)
5495 {
5496         return TRUE;
5497 }
5498
5499 gboolean
5500 mono_gc_is_disabled (void)
5501 {
5502         return FALSE;
5503 }
5504
5505 #ifdef HOST_WIN32
5506 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5507 {
5508         return TRUE;
5509 }
5510 #endif
5511
5512 NurseryClearPolicy
5513 sgen_get_nursery_clear_policy (void)
5514 {
5515         return nursery_clear_policy;
5516 }
5517
5518 MonoVTable*
5519 sgen_get_array_fill_vtable (void)
5520 {
5521         if (!array_fill_vtable) {
5522                 static MonoClass klass;
5523                 static MonoVTable vtable;
5524                 gsize bmap;
5525
5526                 MonoDomain *domain = mono_get_root_domain ();
5527                 g_assert (domain);
5528
5529                 klass.element_class = mono_defaults.byte_class;
5530                 klass.rank = 1;
5531                 klass.instance_size = sizeof (MonoArray);
5532                 klass.sizes.element_size = 1;
5533                 klass.name = "array_filler_type";
5534
5535                 vtable.klass = &klass;
5536                 bmap = 0;
5537                 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5538                 vtable.rank = 1;
5539
5540                 array_fill_vtable = &vtable;
5541         }
5542         return array_fill_vtable;
5543 }
5544
5545 void
5546 sgen_gc_lock (void)
5547 {
5548         LOCK_GC;
5549 }
5550
5551 void
5552 sgen_gc_unlock (void)
5553 {
5554         UNLOCK_GC;
5555 }
5556
5557 void
5558 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5559 {
5560         major_collector.iterate_live_block_ranges (callback);
5561 }
5562
5563 void
5564 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5565 {
5566         major_collector.scan_card_table (FALSE, queue);
5567 }
5568
5569 SgenMajorCollector*
5570 sgen_get_major_collector (void)
5571 {
5572         return &major_collector;
5573 }
5574
5575 void mono_gc_set_skip_thread (gboolean skip)
5576 {
5577         SgenThreadInfo *info = mono_thread_info_current ();
5578
5579         LOCK_GC;
5580         info->gc_disabled = skip;
5581         UNLOCK_GC;
5582 }
5583
5584 SgenRemeberedSet*
5585 sgen_get_remset (void)
5586 {
5587         return &remset;
5588 }
5589
5590 guint
5591 mono_gc_get_vtable_bits (MonoClass *class)
5592 {
5593         if (sgen_need_bridge_processing () && sgen_is_bridge_class (class))
5594                 return SGEN_GC_BIT_BRIDGE_OBJECT;
5595         return 0;
5596 }
5597
5598 void
5599 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5600 {
5601         // FIXME:
5602 }
5603
5604
5605 void
5606 sgen_check_whole_heap_stw (void)
5607 {
5608         sgen_stop_world (0);
5609         sgen_clear_nursery_fragments ();
5610         sgen_check_whole_heap (FALSE);
5611         sgen_restart_world (0, NULL);
5612 }
5613
5614 void
5615 sgen_gc_event_moves (void)
5616 {
5617         if (moved_objects_idx) {
5618                 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5619                 moved_objects_idx = 0;
5620         }
5621 }
5622
5623 #endif /* HAVE_SGEN_GC */