* Author:
* Paolo Molaro (lupus@ximian.com)
*
- * Copyright 2005-2009 Novell, Inc (http://www.novell.com)
+ * Copyright 2005-2010 Novell, Inc (http://www.novell.com)
*
* Thread start/stop adapted from Boehm's GC:
* Copyright (c) 1994 by Xerox Corporation. All rights reserved.
* provided the above notices are retained, and a notice that the code was
* modified is included with the above copyright notice.
*
- * All the rest of the code is LGPL.
+ *
+ * Copyright 2001-2003 Ximian, Inc
+ * Copyright 2003-2010 Novell, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
*
* Important: allocation provides always zeroed memory, having to do
* a memset after allocation is deadly for performance.
* 1) 2-word header with no GC pointers in it (first vtable, second to store the
* forwarding ptr)
* 2) gc descriptor is the second word in the vtable (first word in the class)
- * 3) 8 byte alignment is the minimum and enough (not true for special structures, FIXME)
+ * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
* 4) there is a function to get an object's size and the number of
* elements in an array.
* 5) we know the special way bounds are allocated for complex arrays
+ * 6) we know about proxies and how to treat them when domains are unloaded
*
* Always try to keep stack usage to a minimum: no recursive behaviour
* and no large stack allocs.
* When the nursery is full we start a nursery collection: this is performed with a
* copying GC.
* When the old generation is full we start a copying GC of the old generation as well:
- * this will be changed to mark/compact in the future.
+ * this will be changed to mark&sweep with copying when fragmentation becomes to severe
+ * in the future. Maybe we'll even do both during the same collection like IMMIX.
+ *
* The things that complicate this description are:
* *) pinned objects: we can't move them so we need to keep track of them
* *) no precise info of the thread stacks and registers: we need to be able to
/*
* TODO:
- *) change the jit to emit write barrier calls when needed (we
- can have specialized write barriers): done with icalls, still need to
- use some specialized barriers
+
*) we could have a function pointer in MonoClass to implement
customized write barriers for value types
- *) the write barrier code could be isolated in a couple of functions: when a
- thread is stopped if it's inside the barrier it is let go again
- until we stop outside of them (not really needed, see below GC-safe points)
+
*) investigate the stuff needed to advance a thread to a GC-safe
- point (single-stepping, read from unmapped memory etc) and implement it
- Not needed yet: since we treat the objects reachable from the stack/regs as
- roots, we store the ptr and exec the write barrier so there is no race.
- We may need this to solve the issue with setting the length of arrays and strings.
+ point (single-stepping, read from unmapped memory etc) and implement it.
+ This would enable us to inline allocations and write barriers, for example,
+ or at least parts of them, like the write barrier checks.
We may need this also for handling precise info on stacks, even simple things
as having uninitialized data on the stack and having to wait for the prolog
to zero it. Not an issue for the last frame that we scan conservatively.
We could always not trust the value in the slots anyway.
- *) make the jit info table lock free
+
*) modify the jit to save info about references in stack locations:
this can be done just for locals as a start, so that at least
part of the stack is handled precisely.
- *) Make the debug printf stuff thread and signal safe.
- *) test/fix 64 bit issues
+
*) test/fix endianess issues
- *) port to non-Linux
- *) add batch moving profile info
- *) add more timing info
- *) there is a possible race when an array or string is created: the vtable is set,
- but the length is set only later so if the GC needs to scan the object in that window,
- it won't get the correct size for the object. The object can't have references and it will
- be pinned, but a free memory fragment may be created that overlaps with it.
- We should change the array max_length field to be at the same offset as the string length:
- this way we can have a single special alloc function for them that sets the length.
- Multi-dim arrays have the same issue for rank == 1 for the bounds data.
- *) implement a card table as the write barrier instead of remembered sets?
- *) some sort of blacklist support?
- *) fin_ready_list and critical_fin_list are part of the root set, too
- *) consider lowering the large object min size to 16/32KB or so and benchmark
- *) once mark-compact is implemented we could still keep the
- copying collector for the old generation and use it if we think
- it is better (small heaps and no pinning object in the old
- generation)
- *) avoid the memory store from copy_object when not needed.
- *) optimize the write barriers fastpath to happen in managed code
+
+ *) Implement a card table as the write barrier instead of remembered
+ sets? Card tables are not easy to implement with our current
+ memory layout. We have several different kinds of major heap
+ objects: Small objects in regular blocks, small objects in pinned
+ chunks and LOS objects. If we just have a pointer we have no way
+ to tell which kind of object it points into, therefore we cannot
+ know where its card table is. The least we have to do to make
+ this happen is to get rid of write barriers for indirect stores.
+ (See next item)
+
+ *) Get rid of write barriers for indirect stores. We can do this by
+ telling the GC to wbarrier-register an object once we do an ldloca
+ or ldelema on it, and to unregister it once it's not used anymore
+ (it can only travel downwards on the stack). The problem with
+ unregistering is that it needs to happen eventually no matter
+ what, even if exceptions are thrown, the thread aborts, etc.
+ Rodrigo suggested that we could do only the registering part and
+ let the collector find out (pessimistically) when it's safe to
+ unregister, namely when the stack pointer of the thread that
+ registered the object is higher than it was when the registering
+ happened. This might make for a good first implementation to get
+ some data on performance.
+
+ *) Some sort of blacklist support? Blacklists is a concept from the
+ Boehm GC: if during a conservative scan we find pointers to an
+ area which we might use as heap, we mark that area as unusable, so
+ pointer retention by random pinning pointers is reduced.
+
+ *) experiment with max small object size (very small right now - 2kb,
+ because it's tied to the max freelist size)
+
*) add an option to mmap the whole heap in one chunk: it makes for many
simplifications in the checks (put the nursery at the top and just use a single
check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
was written to, munmap is needed, but the following mmap may not find the same segment
free...)
- *) memzero the fragments after restarting the world and optionally a smaller chunk at a time
- *) an additional strategy to realloc/expand the nursery when fully pinned is to start
- allocating objects in the old generation. This means that we can't optimize away write
- barrier calls in ctors (but that is not valid for other reasons, too).
- *) add write barriers to the Clone methods
+
+ *) memzero the major fragments after restarting the world and optionally a smaller
+ chunk at a time
+
+ *) investigate having fragment zeroing threads
+
+ *) separate locks for finalization and other minor stuff to reduce
+ lock contention
+
+ *) try a different copying order to improve memory locality
+
+ *) a thread abort after a store but before the write barrier will
+ prevent the write barrier from executing
+
+ *) specialized dynamically generated markers/copiers
+
+ *) Dynamically adjust TLAB size to the number of threads. If we have
+ too many threads that do allocation, we might need smaller TLABs,
+ and we might get better performance with larger TLABs if we only
+ have a handful of threads. We could sum up the space left in all
+ assigned TLABs and if that's more than some percentage of the
+ nursery size, reduce the TLAB size.
+
+ *) Explore placing unreachable objects on unused nursery memory.
+ Instead of memset'ng a region to zero, place an int[] covering it.
+ A good place to start is add_nursery_frag. The tricky thing here is
+ placing those objects atomically outside of a collection.
+
+
*/
#include "config.h"
#ifdef HAVE_SGEN_GC
#include <signal.h>
#include <errno.h>
#include <assert.h>
+#include <pthread.h>
#include "metadata/metadata-internals.h"
#include "metadata/class-internals.h"
#include "metadata/gc-internal.h"
#include "metadata/monitor.h"
#include "metadata/threadpool-internals.h"
#include "metadata/mempool-internals.h"
+#include "metadata/marshal.h"
#include "utils/mono-mmap.h"
#include "utils/mono-time.h"
#include "utils/mono-semaphore.h"
/* If not null, dump the heap after each collection into this file */
static FILE *heap_dump_file = NULL;
/* If set, mark stacks conservatively, even if precise marking is possible */
-static gboolean conservative_stack_mark = FALSE;
+static gboolean conservative_stack_mark = TRUE;
+/* If set, do a plausibility check on the scan_starts before and after
+ each collection */
+static gboolean do_scan_starts_check = FALSE;
/*
* Turning on heavy statistics will turn off the managed allocator and
#endif
#ifdef HEAVY_STATISTICS
-static long stat_objects_alloced = 0;
-static long stat_copy_object_called_nursery = 0;
-static long stat_objects_copied_nursery = 0;
-static long stat_copy_object_called_major = 0;
-static long stat_objects_copied_major = 0;
-
-static long stat_copy_object_failed_from_space = 0;
-static long stat_copy_object_failed_forwarded = 0;
-static long stat_copy_object_failed_pinned = 0;
-static long stat_copy_object_failed_large_pinned = 0;
-static long stat_copy_object_failed_to_space = 0;
-
-static long stat_store_remsets = 0;
-static long stat_store_remsets_unique = 0;
-static long stat_saved_remsets_1 = 0;
-static long stat_saved_remsets_2 = 0;
-static long stat_global_remsets_added = 0;
-static long stat_global_remsets_processed = 0;
-
-static long num_copy_object_called = 0;
-static long num_objects_copied = 0;
+static long long stat_objects_alloced = 0;
+static long long stat_bytes_alloced = 0;
+static long long stat_objects_alloced_degraded = 0;
+static long long stat_bytes_alloced_degraded = 0;
+static long long stat_bytes_alloced_los = 0;
+
+static long long stat_copy_object_called_nursery = 0;
+static long long stat_objects_copied_nursery = 0;
+static long long stat_copy_object_called_major = 0;
+static long long stat_objects_copied_major = 0;
+
+static long long stat_scan_object_called_nursery = 0;
+static long long stat_scan_object_called_major = 0;
+
+static long long stat_nursery_copy_object_failed_from_space = 0;
+static long long stat_nursery_copy_object_failed_forwarded = 0;
+static long long stat_nursery_copy_object_failed_pinned = 0;
+
+static long long stat_store_remsets = 0;
+static long long stat_store_remsets_unique = 0;
+static long long stat_saved_remsets_1 = 0;
+static long long stat_saved_remsets_2 = 0;
+static long long stat_global_remsets_added = 0;
+static long long stat_global_remsets_readded = 0;
+static long long stat_global_remsets_processed = 0;
+static long long stat_global_remsets_discarded = 0;
+
+static long long stat_wasted_fragments_used = 0;
+static long long stat_wasted_fragments_bytes = 0;
static int stat_wbarrier_set_field = 0;
static int stat_wbarrier_set_arrayref = 0;
static int stat_wbarrier_object_copy = 0;
#endif
-static long pinned_chunk_bytes_alloced = 0;
-static long large_internal_bytes_alloced = 0;
-
+static long long time_minor_pre_collection_fragment_clear = 0;
+static long long time_minor_pinning = 0;
+static long long time_minor_scan_remsets = 0;
+static long long time_minor_scan_pinned = 0;
+static long long time_minor_scan_registered_roots = 0;
+static long long time_minor_scan_thread_data = 0;
+static long long time_minor_finish_gray_stack = 0;
+static long long time_minor_fragment_creation = 0;
+
+static long long time_major_pre_collection_fragment_clear = 0;
+static long long time_major_pinning = 0;
+static long long time_major_scan_pinned = 0;
+static long long time_major_scan_registered_roots = 0;
+static long long time_major_scan_thread_data = 0;
+static long long time_major_scan_alloc_pinned = 0;
+static long long time_major_scan_finalized = 0;
+static long long time_major_scan_big_objects = 0;
+static long long time_major_finish_gray_stack = 0;
+static long long time_major_sweep = 0;
+static long long time_major_fragment_creation = 0;
+
+static long long pinned_chunk_bytes_alloced = 0;
+static long long large_internal_bytes_alloced = 0;
+
+/* Keep in sync with internal_mem_names in dump_heap()! */
enum {
INTERNAL_MEM_PIN_QUEUE,
INTERNAL_MEM_FRAGMENT,
INTERNAL_MEM_REMSET,
INTERNAL_MEM_GRAY_QUEUE,
INTERNAL_MEM_STORE_REMSET,
+ INTERNAL_MEM_MS_TABLES,
+ INTERNAL_MEM_MS_BLOCK_INFO,
+ INTERNAL_MEM_EPHEMERON_LINK,
INTERNAL_MEM_MAX
};
}
*/
-#define MAX_DEBUG_LEVEL 8
+#define MAX_DEBUG_LEVEL 2
#define DEBUG(level,a) do {if (G_UNLIKELY ((level) <= MAX_DEBUG_LEVEL && (level) <= gc_debug_level)) a;} while (0)
+/* Define this to allow the user to change some of the constants by specifying
+ * their values in the MONO_GC_PARAMS environmental variable. See
+ * mono_gc_base_init for details. */
+#define USER_CONFIG 1
+
#define TV_DECLARE(name) gint64 name
#define TV_GETTIME(tv) tv = mono_100ns_ticks ()
#define TV_ELAPSED(start,end) (int)((end-start) / 10)
+#define TV_ELAPSED_MS(start,end) ((TV_ELAPSED((start),(end)) + 500) / 1000)
+
+#define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
#define GC_BITS_PER_WORD (sizeof (mword) * 8)
struct _LOSObject {
LOSObject *next;
mword size; /* this is the object size */
- int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
guint16 role;
- guint16 scanned;
+ int dummy; /* to have a sizeof (LOSObject) a multiple of ALLOC_ALIGN and data starting at same alignment */
char data [MONO_ZERO_LEN_ARRAY];
};
REMSET_LOCATION, /* just a pointer to the exact location */
REMSET_RANGE, /* range of pointer fields */
REMSET_OBJECT, /* mark all the object for scanning */
- REMSET_OTHER, /* all others */
+ REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
REMSET_TYPE_MASK = 0x3
};
-/* Subtypes of REMSET_OTHER */
-enum {
- REMSET_VTYPE, /* a valuetype array described by a gc descriptor and a count */
- REMSET_ROOT_LOCATION, /* a location inside a root */
-};
-
#ifdef HAVE_KW_THREAD
static __thread RememberedSet *remembered_set MONO_TLS_FAST;
#endif
static pthread_key_t remembered_set_key;
static RememberedSet *global_remset;
static RememberedSet *freed_thread_remsets;
-//static int store_to_global_remset = 0;
static GenericStoreRememberedSet *generic_store_remsets = NULL;
+/*A two slots cache for recently inserted remsets */
+static gpointer global_remset_cache [2];
+
/* FIXME: later choose a size that takes into account the RememberedSet struct
* and doesn't waste any alloc paddin space.
*/
{
MonoClass *klass = ((MonoVTable*)LOAD_VTABLE (o))->klass;
if (klass == mono_defaults.string_class) {
- return sizeof (MonoString) + 2 * mono_string_length ((MonoString*) o) + 2;
+ return sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*) o) + 2;
} else if (klass->rank) {
MonoArray *array = (MonoArray*)o;
- size_t size = sizeof (MonoArray) + mono_array_element_size (klass) * mono_array_length (array);
+ size_t size = sizeof (MonoArray) + klass->sizes.element_size * mono_array_length_fast (array);
if (G_UNLIKELY (array->bounds)) {
size += sizeof (mono_array_size_t) - 1;
size &= ~(sizeof (mono_array_size_t) - 1);
static int num_minor_gcs = 0;
static int num_major_gcs = 0;
+#ifdef USER_CONFIG
+
/* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
-//#define DEFAULT_NURSERY_SIZE (1024*512*125+4096*118)
-#define DEFAULT_NURSERY_SIZE (1024*512*2)
+#define DEFAULT_NURSERY_SIZE (default_nursery_size)
+static int default_nursery_size = (1 << 20);
+#ifdef ALIGN_NURSERY
/* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
+#define DEFAULT_NURSERY_BITS (default_nursery_bits)
+static int default_nursery_bits = 20;
+#endif
+
+#else
+
+#define DEFAULT_NURSERY_SIZE (1024*512*2)
+#ifdef ALIGN_NURSERY
#define DEFAULT_NURSERY_BITS 20
-#define MAJOR_SECTION_SIZE (128*1024)
-#define BLOCK_FOR_OBJECT(o) ((Block*)(((mword)(o)) & ~(MAJOR_SECTION_SIZE - 1)))
-#define MAJOR_SECTION_FOR_OBJECT(o) ((GCMemSection*)BLOCK_FOR_OBJECT ((o)))
-#define MIN_MINOR_COLLECTION_SECTION_ALLOWANCE (DEFAULT_NURSERY_SIZE * 3 / MAJOR_SECTION_SIZE)
-#define DEFAULT_LOS_COLLECTION_TARGET (DEFAULT_NURSERY_SIZE * 2)
+#endif
+
+#endif
+
+#define MIN_LOS_ALLOWANCE (DEFAULT_NURSERY_SIZE * 2)
/* to quickly find the head of an object pinned by a conservative address
* we keep track of the objects allocated for each SCAN_START_SIZE memory
* chunk in the nursery or other memory sections. Larger values have less
#define FREELIST_PAGESIZE 4096
static mword pagesize = 4096;
-static mword nursery_size = DEFAULT_NURSERY_SIZE;
+static mword nursery_size;
static int degraded_mode = 0;
-static int minor_collection_section_allowance = MIN_MINOR_COLLECTION_SECTION_ALLOWANCE;
-static int minor_collection_sections_alloced = 0;
-static int num_major_sections = 0;
-
static LOSObject *los_object_list = NULL;
static mword los_memory_usage = 0;
static mword los_num_objects = 0;
/* use this to tune when to do a major/minor collection */
static mword memory_pressure = 0;
-static GCMemSection *section_list = NULL;
static GCMemSection *nursery_section = NULL;
static mword lowest_heap_address = ~(mword)0;
static mword highest_heap_address = 0;
+static LOCK_DECLARE (interruption_mutex);
+
typedef struct _FinalizeEntry FinalizeEntry;
struct _FinalizeEntry {
FinalizeEntry *next;
int num_links;
};
+typedef struct _EphemeronLinkNode EphemeronLinkNode;
+
+struct _EphemeronLinkNode {
+ EphemeronLinkNode *next;
+ char *array;
+};
+
+typedef struct {
+ void *key;
+ void *value;
+} Ephemeron;
+
#define LARGE_INTERNAL_MEM_HEADER_MAGIC 0x7d289f3a
typedef struct _LargeInternalMemHeader LargeInternalMemHeader;
GENERATION_MAX
};
+int current_collection_generation = -1;
+
/*
* The link pointer is hidden by negating each bit. We use the lowest
* bit of the link (before negation) to store whether it needs
static DisappearingLinkHashTable minor_disappearing_link_hash;
static DisappearingLinkHashTable major_disappearing_link_hash;
+static EphemeronLinkNode *ephemeron_list;
+
static int num_ready_finalizers = 0;
static int no_finalize = 0;
448, 512, 584, 680, 816, 1024, 1360, 2048};
#define FREELIST_NUM_SLOTS (sizeof (freelist_sizes) / sizeof (freelist_sizes [0]))
-static char* max_pinned_chunk_addr = NULL;
-static char* min_pinned_chunk_addr = (char*)-1;
-/* pinned_chunk_list is used for allocations of objects that are never moved */
-static PinnedChunk *pinned_chunk_list = NULL;
+/* This is also the MAJOR_SECTION_SIZE for the copying major
+ collector */
+#define PINNED_CHUNK_SIZE (128 * 1024)
+
/* internal_chunk_list is used for allocating structures needed by the GC */
static PinnedChunk *internal_chunk_list = NULL;
-static gboolean
-obj_is_from_pinned_alloc (char *p)
-{
- return BLOCK_FOR_OBJECT (p)->role == MEMORY_ROLE_PINNED;
-}
-
static int slot_for_size (size_t size);
-static void
-free_pinned_object (PinnedChunk *chunk, char *obj, size_t size)
-{
- void **p = (void**)obj;
- int slot = slot_for_size (size);
-
- g_assert (obj >= (char*)chunk->start_data && obj < ((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE));
- *p = chunk->free_list [slot];
- chunk->free_list [slot] = p;
-}
-
enum {
ROOT_TYPE_NORMAL = 0, /* "normal" roots */
ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */
unsigned int stop_count; /* to catch duplicate signals */
int signal;
int skip;
+ volatile int in_critical_region;
void *stack_end;
void *stack_start;
void *stack_start_limit;
#define REMEMBERED_SET remembered_set
#define STORE_REMSET_BUFFER store_remset_buffer
#define STORE_REMSET_BUFFER_INDEX store_remset_buffer_index
+#define IN_CRITICAL_REGION thread_info->in_critical_region
#else
static pthread_key_t thread_info_key;
#define TLAB_ACCESS_INIT SgenThreadInfo *__thread_info__ = pthread_getspecific (thread_info_key)
#define REMEMBERED_SET (__thread_info__->remset)
#define STORE_REMSET_BUFFER (__thread_info__->store_remset_buffer)
#define STORE_REMSET_BUFFER_INDEX (__thread_info__->store_remset_buffer_index)
+#define IN_CRITICAL_REGION (__thread_info__->in_critical_region)
#endif
+/* we use the memory barrier only to prevent compiler reordering (a memory constraint may be enough) */
+#define ENTER_CRITICAL_REGION do {IN_CRITICAL_REGION = 1;mono_memory_barrier ();} while (0)
+#define EXIT_CRITICAL_REGION do {IN_CRITICAL_REGION = 0;mono_memory_barrier ();} while (0)
+
/*
* FIXME: What is faster, a TLS variable pointing to a structure, or separate TLS
* variables for next+temp_end ?
*/
#ifdef HAVE_KW_THREAD
+static __thread SgenThreadInfo *thread_info;
static __thread char *tlab_start;
static __thread char *tlab_next;
static __thread char *tlab_temp_end;
static char *nursery_next = NULL;
static char *nursery_frag_real_end = NULL;
static char *nursery_real_end = NULL;
-//static char *nursery_first_pinned_start = NULL;
static char *nursery_last_pinned_end = NULL;
/* The size of a TLAB */
*/
static guint32 tlab_size = (1024 * 4);
+/*How much space is tolerable to be wasted from the current fragment when allocating a new TLAB*/
+#define MAX_NURSERY_TLAB_WASTE 512
+
/* fragments that are free and ready to be used for allocation */
static Fragment *nursery_fragments = NULL;
/* freeelist of fragment structures */
static Fragment *fragment_freelist = NULL;
-/*
- * used when moving the objects
- */
-static char *to_space_bumper = NULL;
-static char *to_space_top = NULL;
-static GCMemSection *to_space_section = NULL;
-
/* objects bigger then this go into the large object space */
-#define MAX_SMALL_OBJ_SIZE MAX_FREELIST_SIZE
+#define MAX_SMALL_OBJ_SIZE 2040
/* Functions supplied by the runtime to be called by the GC */
static MonoGCCallbacks gc_callbacks;
+#define ALLOC_ALIGN 8
+#define ALLOC_ALIGN_BITS 3
+
+#define MOVED_OBJECTS_NUM 64
+static void *moved_objects [MOVED_OBJECTS_NUM];
+static int moved_objects_idx = 0;
+
/*
* ######################################################################
* ######## Macros and function declarations.
return (void*)p;
}
+typedef void (*CopyOrMarkObjectFunc) (void**);
+typedef char* (*ScanObjectFunc) (char*);
+
/* forward declarations */
static void* get_internal_mem (size_t size, int type);
static void free_internal_mem (void *addr, int type);
static void* get_os_memory (size_t size, int activate);
+static void* get_os_memory_aligned (mword size, mword alignment, gboolean activate);
static void free_os_memory (void *addr, size_t size);
static G_GNUC_UNUSED void report_internal_mem_usage (void);
static int stop_world (void);
static int restart_world (void);
+static void add_to_global_remset (gpointer ptr);
static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise);
static void scan_from_remsets (void *start_nursery, void *end_nursery);
+static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type);
+static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list);
static void find_pinning_ref_from_thread (char *obj, size_t size);
static void update_current_thread_stack (void *start);
-static GCMemSection* alloc_major_section (void);
-static void finalize_in_range (char *start, char *end, int generation);
+static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
static void add_or_remove_disappearing_link (MonoObject *obj, void **link, gboolean track, int generation);
-static void null_link_in_range (char *start, char *end, int generation);
+static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation);
static void null_links_for_domain (MonoDomain *domain, int generation);
static gboolean search_fragment_for_size (size_t size);
-static void mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end);
+static int search_fragment_for_size_range (size_t desired_size, size_t minimum_size);
+static void build_nursery_fragments (int start_pin, int end_pin);
+static void clear_nursery_fragments (char *next);
+static void pin_from_roots (void *start_nursery, void *end_nursery);
+static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery);
+static void pin_objects_in_section (GCMemSection *section);
+static void optimize_pin_queue (int start_slot);
static void clear_remsets (void);
static void clear_tlabs (void);
-typedef void (*ScanPinnedObjectCallbackFunc) (PinnedChunk*, char*, size_t, void*);
-static void scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data);
-static void sweep_pinned_objects (void);
-static void scan_from_pinned_objects (char *addr_start, char *addr_end);
+typedef void (*IterateObjectCallbackFunc) (char*, size_t, void*);
+static void scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data);
+static char* scan_object (char *start);
+static char* major_scan_object (char *start);
+static void* copy_object_no_checks (void *obj);
+static void copy_object (void **obj_slot);
+static void* get_chunk_freelist (PinnedChunk *chunk, int slot);
+static PinnedChunk* alloc_pinned_chunk (void);
static void free_large_object (LOSObject *obj);
-static void free_major_section (GCMemSection *section);
-static void to_space_expand (void);
+static void sort_addresses (void **array, int size);
+static void drain_gray_stack (void);
+static void finish_gray_stack (char *start_addr, char *end_addr, int generation);
static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track);
void describe_ptr (char *ptr);
-void check_consistency (void);
-char* check_object (char *start);
+static void check_consistency (void);
+static void check_section_scan_starts (GCMemSection *section);
+static void check_scan_starts (void);
+static void check_for_xdomain_refs (void);
+static void dump_occupied (char *start, char *end, char *section_start);
+static void dump_section (GCMemSection *section, const char *type);
+static void dump_heap (const char *type, int num, const char *reason);
+static void commit_stats (int generation);
+static void report_pinned_chunk (PinnedChunk *chunk, int seq);
void mono_gc_scan_for_specific_ref (MonoObject *key);
+static void init_stats (void);
+
+static int mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end);
+static void clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end);
+static void null_ephemerons_for_domain (MonoDomain *domain);
+
+//#define BINARY_PROTOCOL
+#include "sgen-protocol.c"
+#include "sgen-pinning.c"
+#include "sgen-pinning-stats.c"
+#include "sgen-gray.c"
+
/*
* ######################################################################
* ######## GC descriptors
#define VECTOR_INFO_SHIFT 14
#define VECTOR_ELSIZE_SHIFT 3
#define LARGE_BITMAP_SIZE (GC_BITS_PER_WORD - LOW_TYPE_BITS)
-#define MAX_SMALL_SIZE ((1 << SMALL_BITMAP_SHIFT) - 1)
-#define SMALL_SIZE_MASK 0xfff8
#define MAX_ELEMENT_SIZE 0x3ff
-#define ELEMENT_SIZE_MASK (0x3ff << LOW_TYPE_BITS)
#define VECTOR_SUBTYPE_PTRFREE (DESC_TYPE_V_PTRFREE << VECTOR_INFO_SHIFT)
#define VECTOR_SUBTYPE_REFS (DESC_TYPE_V_REFS << VECTOR_INFO_SHIFT)
#define VECTOR_SUBTYPE_RUN_LEN (DESC_TYPE_V_RUN_LEN << VECTOR_INFO_SHIFT)
#define VECTOR_SUBTYPE_BITMAP (DESC_TYPE_V_BITMAP << VECTOR_INFO_SHIFT)
-#define ALLOC_ALIGN 8
-
/* Root bitmap descriptors are simpler: the lower three bits describe the type
* and we either have 30/62 bitmap bits or nibble-based run-length,
static gsize* complex_descriptors = NULL;
static int complex_descriptors_size = 0;
static int complex_descriptors_next = 0;
-static MonoGCMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
+static MonoGCRootMarkFunc user_descriptors [MAX_USER_DESCRIPTORS];
static int user_descriptors_next = 0;
static int
alloc_complex_descriptor (gsize *bitmap, int numbits)
{
- int nwords = numbits/GC_BITS_PER_WORD + 2;
- int res;
- int i;
+ int nwords, res, i;
+
+ numbits = ALIGN_TO (numbits, GC_BITS_PER_WORD);
+ nwords = numbits / GC_BITS_PER_WORD + 1;
LOCK_GC;
res = complex_descriptors_next;
/* helper macros to scan and traverse objects, macros because we resue them in many functions */
#define STRING_SIZE(size,str) do { \
- (size) = sizeof (MonoString) + 2 * (mono_string_length ((MonoString*)(str)) + 1); \
+ (size) = sizeof (MonoString) + 2 * mono_string_length_fast ((MonoString*)(str)) + 2; \
(size) += (ALLOC_ALIGN - 1); \
(size) &= ~(ALLOC_ALIGN - 1); \
} while (0)
int mbwords = (*mbitmap_data++) - 1; \
int el_size = mono_array_element_size (((MonoObject*)(obj))->vtable->klass); \
char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
- char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
+ char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
if (0) { \
MonoObject *myobj = (MonoObject*)start; \
g_print ("found %d at %p (0x%zx): %s.%s\n", mbwords, (obj), (vt)->desc, myobj->vtable->klass->name_space, myobj->vtable->klass->name); \
int etype = (vt)->desc & 0xc000; \
if (etype == (DESC_TYPE_V_REFS << 14)) { \
void **p = (void**)((char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector)); \
- void **end_refs = (void**)((char*)p + el_size * mono_array_length ((MonoArray*)(obj))); \
+ void **end_refs = (void**)((char*)p + el_size * mono_array_length_fast ((MonoArray*)(obj))); \
/* Note: this code can handle also arrays of struct with only references in them */ \
while (p < end_refs) { \
HANDLE_PTR (p, (obj)); \
int offset = ((vt)->desc >> 16) & 0xff; \
int num_refs = ((vt)->desc >> 24) & 0xff; \
char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
- char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
+ char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
while (e_start < e_end) { \
void **p = (void**)e_start; \
int i; \
} \
} else if (etype == DESC_TYPE_V_BITMAP << 14) { \
char *e_start = (char*)(obj) + G_STRUCT_OFFSET (MonoArray, vector); \
- char *e_end = e_start + el_size * mono_array_length ((MonoArray*)(obj)); \
+ char *e_end = e_start + el_size * mono_array_length_fast ((MonoArray*)(obj)); \
while (e_start < e_end) { \
void **p = (void**)e_start; \
gsize _bmap = (vt)->desc >> 16; \
} \
} while (0)
-#define COUNT_OBJECT_TYPES do { \
- switch (desc & 0x7) { \
- case DESC_TYPE_STRING: type_str++; break; \
- case DESC_TYPE_RUN_LENGTH: type_rlen++; break; \
- case DESC_TYPE_ARRAY: case DESC_TYPE_VECTOR: type_vector++; break; \
- case DESC_TYPE_SMALL_BITMAP: type_bitmap++; break; \
- case DESC_TYPE_LARGE_BITMAP: type_lbit++; break; \
- case DESC_TYPE_COMPLEX: type_complex++; break; \
- case DESC_TYPE_COMPLEX_ARR: type_complex++; break; \
- default: g_assert_not_reached (); \
- } \
- } while (0)
-
-
-/*
- * ######################################################################
- * ######## Detecting and removing garbage.
- * ######################################################################
- * This section of code deals with detecting the objects no longer in use
- * and reclaiming the memory.
- */
-
-#if 0
-static mword new_obj_references = 0;
-static mword obj_references_checked = 0;
-
-#undef HANDLE_PTR
-#define HANDLE_PTR(ptr,obj) do { \
- if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
- new_obj_references++; \
- /*printf ("bogus ptr %p found at %p in object %p (%s.%s)\n", *(ptr), (ptr), o, o->vtable->klass->name_space, o->vtable->klass->name);*/ \
- } else { \
- obj_references_checked++; \
- } \
- } while (0)
-
-static void __attribute__((noinline))
-scan_area (char *start, char *end)
-{
- GCVTable *vt;
- int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
- new_obj_references = 0;
- obj_references_checked = 0;
- while (start < end) {
- if (!*(void**)start) {
- start += sizeof (void*); /* should be ALLOC_ALIGN, really */
- continue;
- }
- vt = (GCVTable*)LOAD_VTABLE (start);
- DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
- if (0) {
- MonoObject *obj = (MonoObject*)start;
- g_print ("found at %p (0x%zx): %s.%s\n", start, vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
- }
-
-#define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
-#include "sgen-scan-object.h"
- }
- /*printf ("references to new nursery %p-%p (size: %dk): %d, checked: %d\n", old_start, end, (end-old_start)/1024, new_obj_references, obj_references_checked);
- printf ("\tstrings: %d, runl: %d, vector: %d, bitmaps: %d, lbitmaps: %d, complex: %d\n",
- type_str, type_rlen, type_vector, type_bitmap, type_lbit, type_complex);*/
-}
-#endif
+#include "sgen-major-copying.c"
+//#include "sgen-marksweep.c"
static gboolean
is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain)
}
static void
-scan_area_for_specific_ref (char *start, char *end, MonoObject *key)
+scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data)
{
while (start < end) {
+ size_t size;
if (!*(void**)start) {
start += sizeof (void*); /* should be ALLOC_ALIGN, really */
continue;
}
- start = scan_object_for_specific_ref (start, key);
+ size = safe_object_get_size ((MonoObject*) start);
+ size += ALLOC_ALIGN - 1;
+ size &= ~(ALLOC_ALIGN - 1);
+
+ callback (start, size, data);
+
+ start += size;
}
}
static void
-scan_pinned_object_for_specific_ref_callback (PinnedChunk *chunk, char *obj, size_t size, MonoObject *key)
+scan_object_for_specific_ref_callback (char *obj, size_t size, MonoObject *key)
{
scan_object_for_specific_ref (obj, key);
}
static MonoObject *check_key = NULL;
static RootRecord *check_root = NULL;
-static void*
-check_root_obj_specific_ref_from_marker (void *obj)
+static void
+check_root_obj_specific_ref_from_marker (void **obj)
{
- check_root_obj_specific_ref (check_root, check_key, obj);
- return obj;
+ check_root_obj_specific_ref (check_root, check_key, *obj);
}
static void
break;
}
case ROOT_DESC_USER: {
- MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
+ MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
marker (start_root, check_root_obj_specific_ref_from_marker);
break;
}
void
mono_gc_scan_for_specific_ref (MonoObject *key)
{
- GCMemSection *section;
LOSObject *bigobj;
RootRecord *root;
int i;
- for (section = section_list; section; section = section->block.next)
- scan_area_for_specific_ref (section->data, section->end_data, key);
+ scan_area_with_callback (nursery_section->data, nursery_section->end_data,
+ (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
+
+ major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_specific_ref_callback, key);
for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
scan_object_for_specific_ref (bigobj->data, key);
- scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_pinned_object_for_specific_ref_callback, key);
-
scan_roots_for_specific_ref (key, ROOT_TYPE_NORMAL);
scan_roots_for_specific_ref (key, ROOT_TYPE_WBARRIER);
}
}
+/* Clear all remaining nursery fragments */
+static void
+clear_nursery_fragments (char *next)
+{
+ Fragment *frag;
+ if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
+ g_assert (next <= nursery_frag_real_end);
+ memset (next, 0, nursery_frag_real_end - next);
+ for (frag = nursery_fragments; frag; frag = frag->next) {
+ memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
+ }
+ }
+}
+
static gboolean
need_remove_object_for_domain (char *start, MonoDomain *domain)
{
if (mono_object_domain (start) == domain) {
- DEBUG (1, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
+ DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start));
+ binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
return TRUE;
}
return FALSE;
/* The server could already have been zeroed out, so
we need to check for that, too. */
if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
- DEBUG (1, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
+ DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n",
start, server));
((MonoRealProxy*)start)->unwrapped_server = NULL;
}
}
}
-static void __attribute__((noinline))
-scan_area_for_domain (MonoDomain *domain, char *start, char *end)
-{
- GCVTable *vt;
- gboolean remove;
-
- while (start < end) {
- if (!*(void**)start) {
- start += sizeof (void*); /* should be ALLOC_ALIGN, really */
- continue;
- }
- vt = (GCVTable*)LOAD_VTABLE (start);
- process_object_for_domain_clearing (start, domain);
- remove = need_remove_object_for_domain (start, domain);
- if (remove && ((MonoObject*)start)->synchronisation) {
- void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)start);
- if (dislink)
- mono_gc_register_disappearing_link (NULL, dislink, FALSE);
- }
-
-#define SCAN_OBJECT_NOSCAN
-#define SCAN_OBJECT_ACTION do { \
- if (remove) memset (start, 0, skip_size); \
- } while (0)
-#include "sgen-scan-object.h"
- }
-}
-
static MonoDomain *check_domain = NULL;
-static void*
-check_obj_not_in_domain (void *o)
+static void
+check_obj_not_in_domain (void **o)
{
- g_assert (((MonoObject*)o)->vtable->domain != check_domain);
- return o;
+ g_assert (((MonoObject*)(*o))->vtable->domain != check_domain);
}
static void
break;
}
case ROOT_DESC_USER: {
- MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
+ MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
marker (start_root, check_obj_not_in_domain);
break;
}
}
static void
-clear_domain_process_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
+scan_pinned_object_for_xdomain_refs_callback (char *obj, size_t size, gpointer dummy)
{
- process_object_for_domain_clearing (obj, domain);
+ scan_object_for_xdomain_refs (obj);
}
static void
-clear_domain_free_pinned_object_callback (PinnedChunk *chunk, char *obj, size_t size, MonoDomain *domain)
+check_for_xdomain_refs (void)
{
- if (need_remove_object_for_domain (obj, domain))
- free_pinned_object (chunk, obj, size);
+ LOSObject *bigobj;
+
+ scan_area_for_xdomain_refs (nursery_section->data, nursery_section->end_data);
+
+ major_iterate_objects (TRUE, TRUE, scan_pinned_object_for_xdomain_refs_callback, NULL);
+
+ for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
+ scan_object_for_xdomain_refs (bigobj->data);
}
-static void
-scan_pinned_object_for_xdomain_refs_callback (PinnedChunk *chunk, char *obj, size_t size, gpointer dummy)
+static gboolean
+clear_domain_process_object (char *obj, MonoDomain *domain)
{
- scan_object_for_xdomain_refs (obj);
+ gboolean remove;
+
+ process_object_for_domain_clearing (obj, domain);
+ remove = need_remove_object_for_domain (obj, domain);
+
+ if (remove && ((MonoObject*)obj)->synchronisation) {
+ void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
+ if (dislink)
+ mono_gc_register_disappearing_link (NULL, dislink, FALSE);
+ }
+
+ return remove;
}
static void
-check_for_xdomain_refs (void)
+clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
{
- GCMemSection *section;
- LOSObject *bigobj;
+ if (clear_domain_process_object (obj, domain))
+ memset (obj, 0, size);
+}
- for (section = section_list; section; section = section->block.next)
- scan_area_for_xdomain_refs (section->data, section->end_data);
+static void
+clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
+{
+ clear_domain_process_object (obj, domain);
+}
- for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
- scan_object_for_xdomain_refs (bigobj->data);
+static void
+clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
+{
+ if (need_remove_object_for_domain (obj, domain))
+ major_free_non_pinned_object (obj, size);
+}
- scan_pinned_objects (scan_pinned_object_for_xdomain_refs_callback, NULL);
+static void
+clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
+{
+ if (need_remove_object_for_domain (obj, domain))
+ free_pinned_object (obj, size);
}
/*
void
mono_gc_clear_domain (MonoDomain * domain)
{
- GCMemSection *section;
LOSObject *bigobj, *prev;
- Fragment *frag;
int i;
LOCK_GC;
- /* Clear all remaining nursery fragments */
- if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
- g_assert (nursery_next <= nursery_frag_real_end);
- memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
- for (frag = nursery_fragments; frag; frag = frag->next) {
- memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
- }
- }
+
+ clear_nursery_fragments (nursery_next);
if (xdomain_checks && domain != mono_get_root_domain ()) {
scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
check_for_xdomain_refs ();
}
- for (section = section_list; section; section = section->block.next) {
- scan_area_for_domain (domain, section->data, section->end_data);
- }
+ scan_area_with_callback (nursery_section->data, nursery_section->end_data,
+ (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain);
+
+ /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
+ to memory returned to the OS.*/
+ null_ephemerons_for_domain (domain);
+
+ for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
+ null_links_for_domain (domain, i);
- /* We need two passes over pinned and large objects because
- freeing such an object gives its memory back to the OS (in
- the case of large objects) or obliterates its vtable
- (pinned objects), but we might need to dereference a
- pointer from an object to another object if the first
- object is a proxy. */
- scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_process_pinned_object_callback, domain);
+ /* We need two passes over major and large objects because
+ freeing such objects might give their memory back to the OS
+ (in the case of large objects) or obliterate its vtable
+ (pinned objects with major-copying or pinned and non-pinned
+ objects with major-mark&sweep), but we might need to
+ dereference a pointer from an object to another object if
+ the first object is a proxy. */
+ major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
- process_object_for_domain_clearing (bigobj->data, domain);
+ clear_domain_process_object (bigobj->data, domain);
prev = NULL;
for (bigobj = los_object_list; bigobj;) {
else
los_object_list = bigobj->next;
bigobj = bigobj->next;
- DEBUG (1, fprintf (gc_debug_file, "Freeing large object %p\n",
+ DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n",
bigobj->data));
free_large_object (to_free);
continue;
prev = bigobj;
bigobj = bigobj->next;
}
- scan_pinned_objects ((ScanPinnedObjectCallbackFunc)clear_domain_free_pinned_object_callback, domain);
-
- for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
- null_links_for_domain (domain, i);
+ major_iterate_objects (TRUE, FALSE, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
+ major_iterate_objects (FALSE, TRUE, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
UNLOCK_GC;
}
+static void
+global_remset_cache_clear (void)
+{
+ memset (global_remset_cache, 0, sizeof (global_remset_cache));
+}
+
+/*
+ * Tries to check if a given remset location was already added to the global remset.
+ * It can
+ *
+ * A 2 entry, LRU cache of recently saw location remsets.
+ *
+ * It's hand-coded instead of done using loops to reduce the number of memory references on cache hit.
+ *
+ * Returns TRUE is the element was added..
+ */
+static gboolean
+global_remset_location_was_not_added (gpointer ptr)
+{
+
+ gpointer first = global_remset_cache [0], second;
+ if (first == ptr) {
+ HEAVY_STAT (++stat_global_remsets_discarded);
+ return FALSE;
+ }
+
+ second = global_remset_cache [1];
+
+ if (second == ptr) {
+ /*Move the second to the front*/
+ global_remset_cache [0] = second;
+ global_remset_cache [1] = first;
+
+ HEAVY_STAT (++stat_global_remsets_discarded);
+ return FALSE;
+ }
+
+ global_remset_cache [0] = second;
+ global_remset_cache [1] = ptr;
+ return TRUE;
+}
+
/*
* add_to_global_remset:
*
* a minor collection. This can happen if the objects they point to are pinned.
*/
static void
-add_to_global_remset (gpointer ptr, gboolean root)
+add_to_global_remset (gpointer ptr)
{
RememberedSet *rs;
- DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
-
- g_assert (!root);
g_assert (!ptr_in_nursery (ptr) && ptr_in_nursery (*(gpointer*)ptr));
+ if (!global_remset_location_was_not_added (ptr))
+ return;
+
+ DEBUG (8, fprintf (gc_debug_file, "Adding global remset for %p\n", ptr));
+ binary_protocol_global_remset (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
+
HEAVY_STAT (++stat_global_remsets_added);
/*
* To avoid uncontrolled growth of the global remset, only add each pointer once.
*/
if (global_remset->store_next + 3 < global_remset->end_set) {
- if (root) {
- *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
- *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
- } else {
- *(global_remset->store_next++) = (mword)ptr;
- }
+ *(global_remset->store_next++) = (mword)ptr;
return;
}
rs = alloc_remset (global_remset->end_set - global_remset->data, NULL);
rs->next = global_remset;
global_remset = rs;
- if (root) {
- *(global_remset->store_next++) = (mword)ptr | REMSET_OTHER;
- *(global_remset->store_next++) = (mword)REMSET_ROOT_LOCATION;
- } else {
- *(global_remset->store_next++) = (mword)ptr;
- }
+ *(global_remset->store_next++) = (mword)ptr;
{
int global_rs_size = 0;
}
}
-#include "sgen-gray.c"
-
/*
- * This is how the copying happens from the nursery to the old generation.
- * We assume that at this time all the pinned objects have been identified and
- * marked as such.
- * We run scan_object() for each pinned object so that each referenced
- * objects if possible are copied. The new gray objects created can have
- * scan_object() run on them right away, too.
- * Then we run copy_object() for the precisely tracked roots. At this point
- * all the roots are either gray or black. We run scan_object() on the gray
- * objects until no more gray objects are created.
- * At the end of the process we walk again the pinned list and we unmark
- * the pinned flag. As we go we also create the list of free space for use
- * in the next allocation runs.
- *
- * We need to remember objects from the old generation that point to the new one
- * (or just addresses?).
- *
- * copy_object could be made into a macro once debugged (use inline for now).
+ * FIXME: allocate before calling this function and pass the
+ * destination address.
*/
-
-static char* __attribute__((noinline))
-copy_object (char *obj, char *from_space_start, char *from_space_end)
+static void*
+copy_object_no_checks (void *obj)
{
- static void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
+ static const void *copy_labels [] = { &&LAB_0, &&LAB_1, &&LAB_2, &&LAB_3, &&LAB_4, &&LAB_5, &&LAB_6, &&LAB_7, &&LAB_8 };
- char *forwarded;
mword objsize;
- MonoVTable *vt;
-
- HEAVY_STAT (++num_copy_object_called);
-
- if (!(obj >= from_space_start && obj < from_space_end)) {
- DEBUG (9, fprintf (gc_debug_file, "Not copying %p because it's not in from space (%p-%p)\n",
- obj, from_space_start, from_space_end));
- HEAVY_STAT (++stat_copy_object_failed_from_space);
- return obj;
- }
-
- DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p", obj));
-
- /*
- * obj must belong to one of:
- *
- * 1. the nursery
- * 2. the LOS
- * 3. a pinned chunk
- * 4. a non-to-space section of the major heap
- * 5. a to-space section of the major heap
- *
- * In addition, objects in 1, 2 and 4 might also be pinned.
- * Objects in 1 and 4 might be forwarded.
- *
- * Before we can copy the object we must make sure that we are
- * allowed to, i.e. that the object not pinned, not already
- * forwarded and doesn't belong to the LOS, a pinned chunk, or
- * a to-space section.
- *
- * We are usually called for to-space objects (5) when we have
- * two remset entries for the same reference. The first entry
- * copies the object and updates the reference and the second
- * calls us with the updated reference that points into
- * to-space. There might also be other circumstances where we
- * get to-space objects.
- */
-
- if ((forwarded = object_is_forwarded (obj))) {
- g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
- DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
- HEAVY_STAT (++stat_copy_object_failed_forwarded);
- return forwarded;
- }
- if (object_is_pinned (obj)) {
- g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr);
- DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
- HEAVY_STAT (++stat_copy_object_failed_pinned);
- return obj;
- }
+ char *destination;
+ MonoVTable *vt = ((MonoObject*)obj)->vtable;
+ gboolean has_references = vt->klass->has_references;
objsize = safe_object_get_size ((MonoObject*)obj);
objsize += ALLOC_ALIGN - 1;
objsize &= ~(ALLOC_ALIGN - 1);
- if (ptr_in_nursery (obj))
- goto copy;
-
- /*
- * At this point we know obj is not pinned, not forwarded and
- * belongs to 2, 3, 4, or 5.
- *
- * LOS object (2) are simple, at least until we always follow
- * the rule: if objsize > MAX_SMALL_OBJ_SIZE, pin the object
- * and return it. At the end of major collections, we walk
- * the los list and if the object is pinned, it is marked,
- * otherwise it can be freed.
- *
- * Pinned chunks (3) and major heap sections (4, 5) both
- * reside in blocks, which are always aligned, so once we've
- * eliminated LOS objects, we can just access the block and
- * see whether it's a pinned chunk or a major heap section.
- */
- if (G_UNLIKELY (objsize > MAX_SMALL_OBJ_SIZE || obj_is_from_pinned_alloc (obj))) {
- DEBUG (9, fprintf (gc_debug_file, " (marked LOS/Pinned %p (%s), size: %zd)\n", obj, safe_name (obj), objsize));
- pin_object (obj);
- HEAVY_STAT (++stat_copy_object_failed_large_pinned);
- return obj;
- }
-
- /*
- * Now we know the object is in a major heap section. All we
- * need to do is check whether it's already in to-space (5) or
- * not (4).
- */
- if (MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space) {
- g_assert (objsize <= MAX_SMALL_OBJ_SIZE);
- DEBUG (9, fprintf (gc_debug_file, " (already copied)\n"));
- HEAVY_STAT (++stat_copy_object_failed_to_space);
- return obj;
- }
-
- copy:
- DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", to_space_bumper, ((MonoObject*)obj)->vtable->klass->name, objsize));
+ DEBUG (9, g_assert (vt->klass->inited));
+ MAJOR_GET_COPY_OBJECT_SPACE (destination, objsize, has_references);
- HEAVY_STAT (++num_objects_copied);
-
- /* Make sure we have enough space available */
- if (to_space_bumper + objsize > to_space_top) {
- to_space_expand ();
- g_assert (to_space_bumper + objsize <= to_space_top);
- }
+ DEBUG (9, fprintf (gc_debug_file, " (to %p, %s size: %zd)\n", destination, ((MonoObject*)obj)->vtable->klass->name, objsize));
+ binary_protocol_copy (obj, destination, ((MonoObject*)obj)->vtable, objsize);
if (objsize <= sizeof (gpointer) * 8) {
- mword *dest = (mword*)to_space_bumper;
+ mword *dest = (mword*)destination;
goto *copy_labels [objsize / sizeof (gpointer)];
LAB_8:
(dest) [7] = ((mword*)obj) [7];
{
int ecx;
char* esi = obj;
- char* edi = to_space_bumper;
+ char* edi = destination;
__asm__ __volatile__(
"rep; movsl"
: "=&c" (ecx), "=&D" (edi), "=&S" (esi)
);
}
#else
- memcpy (to_space_bumper, obj, objsize);
+ memcpy (destination, obj, objsize);
#endif
}
/* adjust array->bounds */
- vt = ((MonoObject*)obj)->vtable;
- g_assert (vt->gc_descr);
+ DEBUG (9, g_assert (vt->gc_descr));
if (G_UNLIKELY (vt->rank && ((MonoArray*)obj)->bounds)) {
- MonoArray *array = (MonoArray*)to_space_bumper;
- array->bounds = (MonoArrayBounds*)((char*)to_space_bumper + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
+ MonoArray *array = (MonoArray*)destination;
+ array->bounds = (MonoArrayBounds*)((char*)destination + ((char*)((MonoArray*)obj)->bounds - (char*)obj));
DEBUG (9, fprintf (gc_debug_file, "Array instance %p: size: %zd, rank: %d, length: %d\n", array, objsize, vt->rank, mono_array_length (array)));
}
/* set the forwarding pointer */
- forward_object (obj, to_space_bumper);
- obj = to_space_bumper;
- to_space_section->scan_starts [((char*)obj - (char*)to_space_section->data)/SCAN_START_SIZE] = obj;
- to_space_bumper += objsize;
- DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
- gray_object_enqueue (obj);
- DEBUG (8, g_assert (to_space_bumper <= to_space_top));
+ forward_object (obj, destination);
+ if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
+ if (moved_objects_idx == MOVED_OBJECTS_NUM) {
+ mono_profiler_gc_moves (moved_objects, moved_objects_idx);
+ moved_objects_idx = 0;
+ }
+ moved_objects [moved_objects_idx++] = obj;
+ moved_objects [moved_objects_idx++] = destination;
+ }
+ obj = destination;
+ if (has_references) {
+ DEBUG (9, fprintf (gc_debug_file, "Enqueuing gray object %p (%s)\n", obj, safe_name (obj)));
+ GRAY_OBJECT_ENQUEUE (obj);
+ }
return obj;
}
+/*
+ * This is how the copying happens from the nursery to the old generation.
+ * We assume that at this time all the pinned objects have been identified and
+ * marked as such.
+ * We run scan_object() for each pinned object so that each referenced
+ * objects if possible are copied. The new gray objects created can have
+ * scan_object() run on them right away, too.
+ * Then we run copy_object() for the precisely tracked roots. At this point
+ * all the roots are either gray or black. We run scan_object() on the gray
+ * objects until no more gray objects are created.
+ * At the end of the process we walk again the pinned list and we unmark
+ * the pinned flag. As we go we also create the list of free space for use
+ * in the next allocation runs.
+ *
+ * We need to remember objects from the old generation that point to the new one
+ * (or just addresses?).
+ *
+ * copy_object could be made into a macro once debugged (use inline for now).
+ */
+
+static void __attribute__((noinline))
+copy_object (void **obj_slot)
+{
+ char *forwarded;
+ char *obj = *obj_slot;
+
+ DEBUG (9, g_assert (current_collection_generation == GENERATION_NURSERY));
+
+ HEAVY_STAT (++stat_copy_object_called_nursery);
+
+ if (!ptr_in_nursery (obj)) {
+ HEAVY_STAT (++stat_nursery_copy_object_failed_from_space);
+ return;
+ }
+
+ DEBUG (9, fprintf (gc_debug_file, "Precise copy of %p from %p", obj, obj_slot));
+
+ /*
+ * Before we can copy the object we must make sure that we are
+ * allowed to, i.e. that the object not pinned or not already
+ * forwarded.
+ */
+
+ if ((forwarded = object_is_forwarded (obj))) {
+ DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (already forwarded to %p)\n", forwarded));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_forwarded);
+ *obj_slot = forwarded;
+ return;
+ }
+ if (object_is_pinned (obj)) {
+ DEBUG (9, g_assert (((MonoVTable*)LOAD_VTABLE(obj))->gc_descr));
+ DEBUG (9, fprintf (gc_debug_file, " (pinned, no change)\n"));
+ HEAVY_STAT (++stat_nursery_copy_object_failed_pinned);
+ return;
+ }
+
+ HEAVY_STAT (++stat_objects_copied_nursery);
+
+ *obj_slot = copy_object_no_checks (obj);
+}
+
#undef HANDLE_PTR
#define HANDLE_PTR(ptr,obj) do { \
void *__old = *(ptr); \
void *__copy; \
if (__old) { \
- *(ptr) = __copy = copy_object (__old, from_start, from_end); \
+ copy_object ((ptr)); \
+ __copy = *(ptr); \
DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
- add_to_global_remset ((ptr), FALSE); \
+ add_to_global_remset ((ptr)); \
} \
} while (0)
* Returns a pointer to the end of the object.
*/
static char*
-scan_object (char *start, char* from_start, char* from_end)
+scan_object (char *start)
{
#include "sgen-scan-object.h"
- return start;
-}
-
-/*
- * drain_gray_stack:
- *
- * Scan objects in the gray stack until the stack is empty. This should be called
- * frequently after each object is copied, to achieve better locality and cache
- * usage.
- */
-static void inline
-drain_gray_stack (char *start_addr, char *end_addr)
-{
- char *obj;
+ HEAVY_STAT (++stat_scan_object_called_nursery);
- while ((obj = gray_object_dequeue ())) {
- DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
- scan_object (obj, start_addr, end_addr);
- }
+ return start;
}
/*
return NULL;
}
-#include "sgen-pinning-stats.c"
+#undef HANDLE_PTR
+#define HANDLE_PTR(ptr,obj) do { \
+ void *__old = *(ptr); \
+ void *__copy; \
+ if (__old) { \
+ major_copy_or_mark_object ((ptr)); \
+ __copy = *(ptr); \
+ DEBUG (9, if (__old != __copy) fprintf (gc_debug_file, "Overwrote field at %p with %p (was: %p)\n", (ptr), *(ptr), __old)); \
+ if (G_UNLIKELY (ptr_in_nursery (__copy) && !ptr_in_nursery ((ptr)))) \
+ add_to_global_remset ((ptr)); \
+ } \
+ } while (0)
+
+static char*
+major_scan_object (char *start)
+{
+#include "sgen-scan-object.h"
+
+ HEAVY_STAT (++stat_scan_object_called_major);
+
+ return start;
+}
+
+/*
+ * drain_gray_stack:
+ *
+ * Scan objects in the gray stack until the stack is empty. This should be called
+ * frequently after each object is copied, to achieve better locality and cache
+ * usage.
+ */
+static void inline
+drain_gray_stack (void)
+{
+ char *obj;
+
+ if (current_collection_generation == GENERATION_NURSERY) {
+ for (;;) {
+ GRAY_OBJECT_DEQUEUE (obj);
+ if (!obj)
+ break;
+ DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
+ scan_object (obj);
+ }
+ } else {
+ for (;;) {
+ GRAY_OBJECT_DEQUEUE (obj);
+ if (!obj)
+ break;
+ DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj)));
+ major_scan_object (obj);
+ }
+ }
+}
/*
* Addresses from start to end are already sorted. This function finds
continue;
}
idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
+ g_assert (idx < section->num_scan_start);
search_start = (void*)section->scan_starts [idx];
if (!search_start || search_start > addr) {
while (idx) {
DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size));
if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) {
DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count));
+ binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start));
pin_object (search_start);
+ GRAY_OBJECT_ENQUEUE (search_start);
if (heap_dump_file)
pin_stats_register_object (search_start, last_obj_size);
definitely_pinned [count] = search_start;
return count;
}
-static void** pin_queue;
-static int pin_queue_size = 0;
-static int next_pin_slot = 0;
+static void
+pin_objects_in_section (GCMemSection *section)
+{
+ int start = section->pin_queue_start;
+ int end = section->pin_queue_end;
+ if (start != end) {
+ int reduced_to;
+ reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
+ section->data, section->next_data);
+ section->pin_queue_start = start;
+ section->pin_queue_end = start + reduced_to;
+ }
+}
static int
new_gap (int gap)
return gap;
}
-#if 0
-static int
-compare_addr (const void *a, const void *b)
-{
- return *(const void **)a - *(const void **)b;
-}
-#endif
-
-/* sort the addresses in array in increasing order */
+/* Sort the addresses in array in increasing order.
+ * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
+ */
static void
sort_addresses (void **array, int size)
{
- /*
- * qsort is slower as predicted.
- * qsort (array, size, sizeof (gpointer), compare_addr);
- * return;
- */
- int gap = size;
- int swapped, end;
- while (TRUE) {
- int i;
- gap = new_gap (gap);
- swapped = FALSE;
- end = size - gap;
- for (i = 0; i < end; i++) {
- int j = i + gap;
- if (array [i] > array [j]) {
- void* val = array [i];
- array [i] = array [j];
- array [j] = val;
- swapped = TRUE;
- }
+ int i;
+ void *tmp;
+
+ for (i = 1; i < size; ++i) {
+ int child = i;
+ while (child > 0) {
+ int parent = (child - 1) / 2;
+
+ if (array [parent] >= array [child])
+ break;
+
+ tmp = array [parent];
+ array [parent] = array [child];
+ array [child] = tmp;
+
+ child = parent;
+ }
+ }
+
+ for (i = size - 1; i > 0; --i) {
+ int end, root;
+ tmp = array [i];
+ array [i] = array [0];
+ array [0] = tmp;
+
+ end = i - 1;
+ root = 0;
+
+ while (root * 2 + 1 <= end) {
+ int child = root * 2 + 1;
+
+ if (child < end && array [child] < array [child + 1])
+ ++child;
+ if (array [root] >= array [child])
+ break;
+
+ tmp = array [root];
+ array [root] = array [child];
+ array [child] = tmp;
+
+ root = child;
}
- if (gap == 1 && !swapped)
- break;
}
}
}
-static int
-optimized_pin_queue_search (void *addr)
-{
- int first = 0, last = next_pin_slot;
- while (first < last) {
- int middle = first + ((last - first) >> 1);
- if (addr <= pin_queue [middle])
- last = middle;
- else
- first = middle + 1;
- }
- g_assert (first == last);
- return first;
-}
-
-static void
-find_optimized_pin_queue_area (void *start, void *end, int *first, int *last)
-{
- *first = optimized_pin_queue_search (start);
- *last = optimized_pin_queue_search (end);
-}
-
-static void
-realloc_pin_queue (void)
-{
- int new_size = pin_queue_size? pin_queue_size + pin_queue_size/2: 1024;
- void **new_pin = get_internal_mem (sizeof (void*) * new_size, INTERNAL_MEM_PIN_QUEUE);
- memcpy (new_pin, pin_queue, sizeof (void*) * next_pin_slot);
- free_internal_mem (pin_queue, INTERNAL_MEM_PIN_QUEUE);
- pin_queue = new_pin;
- pin_queue_size = new_size;
- DEBUG (4, fprintf (gc_debug_file, "Reallocated pin queue to size: %d\n", new_size));
-}
-
-#include "sgen-pinning.c"
-
/*
* Scan the memory between start and end and queue values which could be pointers
* to the area between start_nursery and end_nursery for later consideration.
DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count));
}
-/*
- * If generation is 0, just mark objects in the nursery, the others we don't care,
- * since they are not going to move anyway.
- * There are different areas that are scanned for pinned pointers:
- * *) the thread stacks (when jit support is ready only the unmanaged frames)
- * *) the pinned handle table
- * *) the pinned roots
- *
- * Note: when we'll use a write barrier for old to new gen references, we need to
- * keep track of old gen objects that point to pinned new gen objects because in that
- * case the referenced object will be moved maybe at the next collection, but there
- * is no write in the old generation area where the pinned object is referenced
- * and we may not consider it as reachable.
- */
-static G_GNUC_UNUSED void
-mark_pinned_objects (int generation)
-{
-}
-
/*
* Debugging function: find in the conservative roots where @obj is being pinned.
*/
evacuate_pin_staging_area ();
}
-/* Copy function called from user defined mark functions */
-static char *user_copy_n_start;
-static char *user_copy_n_end;
-
-static void*
-user_copy (void *addr)
-{
- if (addr)
- return copy_object (addr, user_copy_n_start, user_copy_n_end);
- else
- return NULL;
-}
-
/*
* The memory area from start_root to end_root contains pointers to objects.
* Their position is precisely described by @desc (this means that the pointer
* This functions copies them to to_space updates them.
*/
static void
-precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
+precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc)
{
switch (desc & ROOT_DESC_TYPE_MASK) {
case ROOT_DESC_BITMAP:
desc >>= ROOT_DESC_TYPE_SHIFT;
while (desc) {
if ((desc & 1) && *start_root) {
- *start_root = copy_object (*start_root, n_start, n_end);
+ copy_func (start_root);
DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root));
- drain_gray_stack (n_start, n_end);
+ drain_gray_stack ();
}
desc >>= 1;
start_root++;
void **objptr = start_run;
while (bmap) {
if ((bmap & 1) && *objptr) {
- *objptr = copy_object (*objptr, n_start, n_end);
+ copy_func (objptr);
DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr));
- drain_gray_stack (n_start, n_end);
+ drain_gray_stack ();
}
bmap >>= 1;
++objptr;
break;
}
case ROOT_DESC_USER: {
- MonoGCMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
-
- user_copy_n_start = n_start;
- user_copy_n_end = n_end;
- marker (start_root, user_copy);
+ MonoGCRootMarkFunc marker = user_descriptors [desc >> ROOT_DESC_TYPE_SHIFT];
+ marker (start_root, copy_func);
break;
}
case ROOT_DESC_RUN_LEN:
/* size must be a power of 2 */
static void*
-get_os_memory_aligned (mword size, gboolean activate)
+get_os_memory_aligned (mword size, mword alignment, gboolean activate)
{
/* Allocate twice the memory to be able to put the block on an aligned address */
- char *mem = get_os_memory (size * 2, activate);
+ char *mem = get_os_memory (size + alignment, activate);
char *aligned;
g_assert (mem);
- aligned = (char*)((mword)(mem + (size - 1)) & ~(size - 1));
- g_assert (aligned >= mem && aligned + size <= mem + size * 2 && !((mword)aligned & (size - 1)));
+ aligned = (char*)((mword)(mem + (alignment - 1)) & ~(alignment - 1));
+ g_assert (aligned >= mem && aligned + size <= mem + size + alignment && !((mword)aligned & (alignment - 1)));
if (aligned > mem)
free_os_memory (mem, aligned - mem);
- if (aligned + size < mem + size * 2)
- free_os_memory (aligned + size, (mem + size * 2) - (aligned + size));
+ if (aligned + size < mem + size + alignment)
+ free_os_memory (aligned + size, (mem + size + alignment) - (aligned + size));
return aligned;
}
g_assert (nursery_size == DEFAULT_NURSERY_SIZE);
alloc_size = nursery_size;
#ifdef ALIGN_NURSERY
- data = get_os_memory_aligned (alloc_size, TRUE);
+ data = get_os_memory_aligned (alloc_size, alloc_size, TRUE);
#else
data = get_os_memory (alloc_size, TRUE);
#endif
section->data = section->next_data = data;
section->size = alloc_size;
section->end_data = nursery_real_end;
- scan_starts = alloc_size / SCAN_START_SIZE;
+ scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
section->num_scan_start = scan_starts;
section->block.role = MEMORY_ROLE_GEN0;
-
- /* add to the section list */
- section->block.next = section_list;
- section_list = section;
+ section->block.next = NULL;
nursery_section = section;
}
static void
-scan_finalizer_entries (FinalizeEntry *list, char *start, char *end) {
+scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeEntry *list) {
FinalizeEntry *fin;
for (fin = list; fin; fin = fin->next) {
if (!fin->object)
continue;
DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)));
- fin->object = copy_object (fin->object, start, end);
+ copy_func (&fin->object);
}
}
-/*
- * Update roots in the old generation. Since we currently don't have the
- * info from the write barriers, we just scan all the objects.
- */
-static G_GNUC_UNUSED void
-scan_old_generation (char *start, char* end)
-{
- GCMemSection *section;
- LOSObject *big_object;
- char *p;
-
- for (section = section_list; section; section = section->block.next) {
- if (section == nursery_section)
- continue;
- DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
- /* we have to deal with zeroed holes in old generation (truncated strings ...) */
- p = section->data;
- while (p < section->next_data) {
- if (!*(void**)p) {
- p += ALLOC_ALIGN;
- continue;
- }
- DEBUG (8, fprintf (gc_debug_file, "Precise old object scan of %p (%s)\n", p, safe_name (p)));
- p = scan_object (p, start, end);
- }
- }
- /* scan the old object space, too */
- for (big_object = los_object_list; big_object; big_object = big_object->next) {
- DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
- scan_object (big_object->data, start, end);
- }
- /* scan the list of objects ready for finalization */
- scan_finalizer_entries (fin_ready_list, start, end);
- scan_finalizer_entries (critical_fin_list, start, end);
-}
-
static mword fragment_total = 0;
/*
* We found a fragment of free memory in the nursery: memzero it and if
{
Fragment *fragment;
DEBUG (4, fprintf (gc_debug_file, "Found empty fragment: %p-%p, size: %zd\n", frag_start, frag_end, frag_size));
+ binary_protocol_empty (frag_start, frag_size);
/* memsetting just the first chunk start is bound to provide better cache locality */
if (nursery_clear_policy == CLEAR_AT_GC)
memset (frag_start, 0, frag_size);
fragment_total += frag_size;
} else {
/* Clear unused fragments, pinning depends on this */
+ /*TODO place an int[] here instead of the memset if size justify it*/
memset (frag_start, 0, frag_size);
}
}
-static int
-scan_needed_big_objects (char *start_addr, char *end_addr)
-{
- LOSObject *big_object;
- int count = 0;
- for (big_object = los_object_list; big_object; big_object = big_object->next) {
- if (!big_object->scanned && object_is_pinned (big_object->data)) {
- DEBUG (5, fprintf (gc_debug_file, "Scan of big object: %p (%s), size: %zd\n", big_object->data, safe_name (big_object->data), big_object->size));
- scan_object (big_object->data, start_addr, end_addr);
- big_object->scanned = TRUE;
- count++;
- }
- }
- return count;
-}
-
static const char*
generation_name (int generation)
{
}
}
-static void
-new_to_space_section (void)
-{
- /* FIXME: if the current to_space_section is empty, we don't
- have to allocate a new one */
-
- to_space_section = alloc_major_section ();
- to_space_bumper = to_space_section->next_data;
- to_space_top = to_space_section->end_data;
-}
-
-static void
-to_space_set_next_data (void)
-{
- g_assert (to_space_bumper >= to_space_section->next_data && to_space_bumper <= to_space_section->end_data);
- to_space_section->next_data = to_space_bumper;
-}
-
-static void
-to_space_expand (void)
-{
- if (to_space_section) {
- g_assert (to_space_top == to_space_section->end_data);
- to_space_set_next_data ();
- }
-
- new_to_space_section ();
-}
-
-static void
-unset_to_space (void)
-{
- /* between collections the to_space_bumper is invalidated
- because degraded allocations might occur, so we set it to
- NULL, just to make it explicit */
- to_space_bumper = NULL;
-
- /* don't unset to_space_section if we implement the FIXME in
- new_to_space_section */
- to_space_section = NULL;
-}
-
-static gboolean
-object_is_in_to_space (char *obj)
-{
- mword objsize;
-
- /* nursery */
- if (ptr_in_nursery (obj))
- return FALSE;
-
- objsize = safe_object_get_size ((MonoObject*)obj);
- objsize += ALLOC_ALIGN - 1;
- objsize &= ~(ALLOC_ALIGN - 1);
-
- /* LOS */
- if (objsize > MAX_SMALL_OBJ_SIZE)
- return FALSE;
-
- /* pinned chunk */
- if (obj_is_from_pinned_alloc (obj))
- return FALSE;
-
- /* now we know it's in a major heap section */
- return MAJOR_SECTION_FOR_OBJECT (obj)->is_to_space;
-}
-
static void
finish_gray_stack (char *start_addr, char *end_addr, int generation)
{
TV_DECLARE (atv);
TV_DECLARE (btv);
- int fin_ready, bigo_scanned_num;
+ int fin_ready;
+ int ephemeron_rounds = 0;
+ CopyOrMarkObjectFunc copy_func = current_collection_generation == GENERATION_NURSERY ? copy_object : major_copy_or_mark_object;
/*
* We copied all the reachable objects. Now it's the time to copy
* To achieve better cache locality and cache usage, we drain the gray stack
* frequently, after each object is copied, and just finish the work here.
*/
- drain_gray_stack (start_addr, end_addr);
+ drain_gray_stack ();
TV_GETTIME (atv);
- //scan_old_generation (start_addr, end_addr);
DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation)));
/* walk the finalization queue and move also the objects that need to be
* finalized: use the finalized objects as new roots so the objects they depend
* that are fin-ready. Speedup with a flag?
*/
do {
+ /*
+ * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
+ * before processing finalizable objects to avoid finalizing reachable values.
+ *
+ * It must be done inside the finalizaters loop since objects must not be removed from CWT tables
+ * while they are been finalized.
+ */
+ int done_with_ephemerons = 0;
+ do {
+ done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr);
+ drain_gray_stack ();
+ ++ephemeron_rounds;
+ } while (!done_with_ephemerons);
+
fin_ready = num_ready_finalizers;
- finalize_in_range (start_addr, end_addr, generation);
+ finalize_in_range (copy_func, start_addr, end_addr, generation);
if (generation == GENERATION_OLD)
- finalize_in_range (nursery_start, nursery_real_end, GENERATION_NURSERY);
- bigo_scanned_num = scan_needed_big_objects (start_addr, end_addr);
+ finalize_in_range (copy_func, nursery_start, nursery_real_end, GENERATION_NURSERY);
/* drain the new stack that might have been created */
DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n"));
- drain_gray_stack (start_addr, end_addr);
- } while (fin_ready != num_ready_finalizers || bigo_scanned_num);
+ drain_gray_stack ();
+ } while (fin_ready != num_ready_finalizers);
+
+ /*
+ * Clear ephemeron pairs with unreachable keys.
+ * We pass the copy func so we can figure out if an array was promoted or not.
+ */
+ clear_unreachable_ephemerons (copy_func, start_addr, end_addr);
+
TV_GETTIME (btv);
- DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs\n", generation_name (generation), TV_ELAPSED (atv, btv)));
+ DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds));
/*
* handle disappearing links
*/
g_assert (gray_object_queue_is_empty ());
for (;;) {
- null_link_in_range (start_addr, end_addr, generation);
+ null_link_in_range (copy_func, start_addr, end_addr, generation);
if (generation == GENERATION_OLD)
- null_link_in_range (start_addr, end_addr, GENERATION_NURSERY);
+ null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY);
if (gray_object_queue_is_empty ())
break;
- drain_gray_stack (start_addr, end_addr);
+ drain_gray_stack ();
}
g_assert (gray_object_queue_is_empty ());
- /* DEBUG (2, fprintf (gc_debug_file, "Copied from %s to old space: %d bytes (%p-%p)\n", generation_name (generation), (int)(to_space_bumper - to_space), to_space, to_space_bumper)); */
- to_space_set_next_data ();
+}
+
+static void
+check_section_scan_starts (GCMemSection *section)
+{
+ int i;
+ for (i = 0; i < section->num_scan_start; ++i) {
+ if (section->scan_starts [i]) {
+ guint size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
+ g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
+ }
+ }
+}
+
+static void
+check_scan_starts (void)
+{
+ if (!do_scan_starts_check)
+ return;
+ check_section_scan_starts (nursery_section);
+ major_check_scan_starts ();
}
static int last_num_pinned = 0;
clear_tlabs ();
}
-/* FIXME: later reduce code duplication here with the above
- * We don't keep track of section fragments for non-nursery sections yet, so
- * just memset to 0.
- */
static void
-build_section_fragments (GCMemSection *section)
-{
- int i;
- char *frag_start, *frag_end;
- size_t frag_size;
-
- /* clear scan starts */
- memset (section->scan_starts, 0, section->num_scan_start * sizeof (gpointer));
- frag_start = section->data;
- section->next_data = section->data;
- for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
- frag_end = pin_queue [i];
- /* remove the pin bit from pinned objects */
- unpin_object (frag_end);
- if (frag_end >= section->data + section->size) {
- frag_end = section->data + section->size;
- } else {
- section->scan_starts [((char*)frag_end - (char*)section->data)/SCAN_START_SIZE] = frag_end;
- }
- frag_size = frag_end - frag_start;
- if (frag_size)
- memset (frag_start, 0, frag_size);
- frag_size = safe_object_get_size ((MonoObject*)pin_queue [i]);
- frag_size += ALLOC_ALIGN - 1;
- frag_size &= ~(ALLOC_ALIGN - 1);
- frag_start = (char*)pin_queue [i] + frag_size;
- section->next_data = MAX (section->next_data, frag_start);
- }
- frag_end = section->end_data;
- frag_size = frag_end - frag_start;
- if (frag_size)
- memset (frag_start, 0, frag_size);
-}
-
-static void
-scan_from_registered_roots (char *addr_start, char *addr_end, int root_type)
+scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type)
{
int i;
RootRecord *root;
for (i = 0; i < roots_hash_size [root_type]; ++i) {
for (root = roots_hash [root_type][i]; root; root = root->next) {
DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", root->start_root, root->end_root, (void*)root->root_desc));
- precisely_scan_objects_from ((void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
+ precisely_scan_objects_from (copy_func, (void**)root->start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc);
}
}
}
fprintf (heap_dump_file, "</section>\n");
}
+static void
+dump_object (MonoObject *obj, gboolean dump_location)
+{
+ static char class_name [1024];
+
+ MonoClass *class = mono_object_class (obj);
+ int i, j;
+
+ /*
+ * Python's XML parser is too stupid to parse angle brackets
+ * in strings, so we just ignore them;
+ */
+ i = j = 0;
+ while (class->name [i] && j < sizeof (class_name) - 1) {
+ if (!strchr ("<>\"", class->name [i]))
+ class_name [j++] = class->name [i];
+ ++i;
+ }
+ g_assert (j < sizeof (class_name));
+ class_name [j] = 0;
+
+ fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"",
+ class->name_space, class_name,
+ safe_object_get_size (obj));
+ if (dump_location) {
+ const char *location;
+ if (ptr_in_nursery (obj))
+ location = "nursery";
+ else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
+ location = "major";
+ else
+ location = "LOS";
+ fprintf (heap_dump_file, " location=\"%s\"", location);
+ }
+ fprintf (heap_dump_file, "/>\n");
+}
+
static void
dump_heap (const char *type, int num, const char *reason)
{
static char const *internal_mem_names [] = { "pin-queue", "fragment", "section", "scan-starts",
"fin-table", "finalize-entry", "dislink-table",
"dislink", "roots-table", "root-record", "statistics",
- "remset", "gray-queue", "store-remset" };
+ "remset", "gray-queue", "store-remset", "marksweep-tables",
+ "marksweep-block-info", "ephemeron-link" };
- GCMemSection *section;
+ ObjectList *list;
LOSObject *bigobj;
int i;
if (reason)
fprintf (heap_dump_file, " reason=\"%s\"", reason);
fprintf (heap_dump_file, ">\n");
- fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%ld\"/>\n", pinned_chunk_bytes_alloced);
- fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%ld\"/>\n", large_internal_bytes_alloced);
+ fprintf (heap_dump_file, "<other-mem-usage type=\"pinned-chunks\" size=\"%lld\"/>\n", pinned_chunk_bytes_alloced);
+ fprintf (heap_dump_file, "<other-mem-usage type=\"large-internal\" size=\"%lld\"/>\n", large_internal_bytes_alloced);
fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
for (i = 0; i < INTERNAL_MEM_MAX; ++i)
fprintf (heap_dump_file, "<other-mem-usage type=\"%s\" size=\"%ld\"/>\n", internal_mem_names [i], small_internal_mem_bytes [i]);
/* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", pinned_byte_counts [PIN_TYPE_OTHER]);
+ fprintf (heap_dump_file, "<pinned-objects>\n");
+ for (list = pinned_objects; list; list = list->next)
+ dump_object (list->obj, TRUE);
+ fprintf (heap_dump_file, "</pinned-objects>\n");
+
dump_section (nursery_section, "nursery");
- for (section = section_list; section; section = section->block.next) {
- if (section != nursery_section)
- dump_section (section, "old");
- }
+ major_dump_heap ();
fprintf (heap_dump_file, "<los>\n");
- for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
- MonoObject *obj = (MonoObject*) bigobj->data;
- MonoClass *class = mono_object_class (obj);
-
- fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%d\"/>\n",
- class->name_space, class->name,
- safe_object_get_size (obj));
- }
+ for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
+ dump_object ((MonoObject*)bigobj->data, FALSE);
fprintf (heap_dump_file, "</los>\n");
fprintf (heap_dump_file, "</collection>\n");
{
static gboolean inited = FALSE;
-#ifdef HEAVY_STATISTICS
- num_copy_object_called = 0;
- num_objects_copied = 0;
-#endif
-
if (inited)
return;
+ mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pre_collection_fragment_clear);
+ mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_pinning);
+ mono_counters_register ("Minor scan remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_remsets);
+ mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_pinned);
+ mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_registered_roots);
+ mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_scan_thread_data);
+ mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_finish_gray_stack);
+ mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_minor_fragment_creation);
+
+ mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pre_collection_fragment_clear);
+ mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_pinning);
+ mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_pinned);
+ mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_registered_roots);
+ mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_thread_data);
+ mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_alloc_pinned);
+ mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_finalized);
+ mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_scan_big_objects);
+ mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_finish_gray_stack);
+ mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_sweep);
+ mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG, &time_major_fragment_creation);
+
#ifdef HEAVY_STATISTICS
mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
mono_counters_register ("# objects allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced);
+ mono_counters_register ("bytes allocated", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced);
+ mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
+ mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
+ mono_counters_register ("bytes allocated in LOS", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_los);
+
mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
- mono_counters_register ("# copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_from_space);
- mono_counters_register ("# copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_forwarded);
- mono_counters_register ("# copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_pinned);
- mono_counters_register ("# copy_object() failed large or pinned chunk", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_large_pinned);
- mono_counters_register ("# copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_failed_to_space);
+ mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
+ mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
+
+ mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
+ mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
+ mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
+
+ mono_counters_register ("# wasted fragments used", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_used);
+ mono_counters_register ("bytes in wasted fragments", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_wasted_fragments_bytes);
mono_counters_register ("Store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets);
mono_counters_register ("Unique store remsets", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_store_remsets_unique);
mono_counters_register ("Saved remsets 1", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_1);
mono_counters_register ("Saved remsets 2", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_saved_remsets_2);
mono_counters_register ("Global remsets added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_added);
+ mono_counters_register ("Global remsets re-added", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_readded);
mono_counters_register ("Global remsets processed", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_processed);
+ mono_counters_register ("Global remsets discarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_global_remsets_discarded);
+
#endif
inited = TRUE;
}
-static void
-commit_stats (int generation)
-{
-#ifdef HEAVY_STATISTICS
- if (generation == GENERATION_NURSERY) {
- stat_copy_object_called_nursery += num_copy_object_called;
- stat_objects_copied_nursery += num_objects_copied;
- } else {
- g_assert (generation == GENERATION_OLD);
- stat_copy_object_called_major += num_copy_object_called;
- stat_objects_copied_major += num_objects_copied;
- }
-#endif
-}
-
/*
* Collect objects in the nursery. Returns whether to trigger a major
* collection.
collect_nursery (size_t requested_size)
{
size_t max_garbage_amount;
- int i;
char *orig_nursery_next;
- Fragment *frag;
- GCMemSection *section;
- int old_num_major_sections = num_major_sections;
- int sections_alloced;
TV_DECLARE (all_atv);
TV_DECLARE (all_btv);
TV_DECLARE (atv);
TV_DECLARE (btv);
+ current_collection_generation = GENERATION_NURSERY;
+
init_stats ();
+ binary_protocol_collection (GENERATION_NURSERY);
+ check_scan_starts ();
degraded_mode = 0;
orig_nursery_next = nursery_next;
/* FIXME: optimize later to use the higher address where an object can be present */
nursery_next = MAX (nursery_next, nursery_real_end);
- if (consistency_check_at_minor_collection)
- check_consistency ();
-
DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", num_minor_gcs, nursery_start, nursery_next, (int)(nursery_next - nursery_start)));
max_garbage_amount = nursery_next - nursery_start;
g_assert (nursery_section->size >= max_garbage_amount);
- /* Clear all remaining nursery fragments, pinning depends on this */
- if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
- g_assert (orig_nursery_next <= nursery_frag_real_end);
- memset (orig_nursery_next, 0, nursery_frag_real_end - orig_nursery_next);
- for (frag = nursery_fragments; frag; frag = frag->next) {
- memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
- }
- }
+ /* world must be stopped already */
+ TV_GETTIME (all_atv);
+ TV_GETTIME (atv);
+
+ /* Pinning depends on this */
+ clear_nursery_fragments (orig_nursery_next);
+
+ TV_GETTIME (btv);
+ time_minor_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
if (xdomain_checks)
check_for_xdomain_refs ();
nursery_section->next_data = nursery_next;
- if (!to_space_section) {
- new_to_space_section ();
- } else {
- /* we might have done degraded allocation since the
- last collection */
- g_assert (to_space_bumper <= to_space_section->next_data);
- to_space_bumper = to_space_section->next_data;
+ major_start_nursery_collection ();
- to_space_section->is_to_space = TRUE;
- }
gray_object_queue_init ();
num_minor_gcs++;
mono_stats.minor_gc_count ++;
- /* world must be stopped already */
- TV_GETTIME (all_atv);
- TV_GETTIME (atv);
+
+ global_remset_cache_clear ();
+
/* pin from pinned handles */
init_pinning ();
pin_from_roots (nursery_start, nursery_next);
/* identify pinned objects */
optimize_pin_queue (0);
next_pin_slot = pin_objects_from_addresses (nursery_section, pin_queue, pin_queue + next_pin_slot, nursery_start, nursery_next);
- TV_GETTIME (btv);
- DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
+ nursery_section->pin_queue_start = 0;
+ nursery_section->pin_queue_end = next_pin_slot;
+ TV_GETTIME (atv);
+ time_minor_pinning += TV_ELAPSED_MS (btv, atv);
+ DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (btv, atv)));
DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
+ if (consistency_check_at_minor_collection)
+ check_consistency ();
+
/*
* walk all the roots and copy the young objects to the old generation,
* starting from to_space
scan_from_remsets (nursery_start, nursery_next);
/* we don't have complete write barrier yet, so we scan all the old generation sections */
- TV_GETTIME (atv);
- DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (btv, atv)));
+ TV_GETTIME (btv);
+ time_minor_scan_remsets += TV_ELAPSED_MS (atv, btv);
+ DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv)));
- /* the pinned objects are roots */
- for (i = 0; i < next_pin_slot; ++i) {
- DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n", i, pin_queue [i], safe_name (pin_queue [i])));
- scan_object (pin_queue [i], nursery_start, nursery_next);
- }
+ drain_gray_stack ();
+
+ TV_GETTIME (atv);
+ time_minor_scan_pinned += TV_ELAPSED_MS (btv, atv);
/* registered roots, this includes static fields */
- scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_NORMAL);
- scan_from_registered_roots (nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
- scan_thread_data (nursery_start, nursery_next, TRUE);
- /* alloc_pinned objects */
- scan_from_pinned_objects (nursery_start, nursery_next);
+ scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_NORMAL);
+ scan_from_registered_roots (copy_object, nursery_start, nursery_next, ROOT_TYPE_WBARRIER);
TV_GETTIME (btv);
- DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (atv, btv)));
+ time_minor_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
+ /* thread data */
+ scan_thread_data (nursery_start, nursery_next, TRUE);
+ TV_GETTIME (atv);
+ time_minor_scan_thread_data += TV_ELAPSED_MS (btv, atv);
+ btv = atv;
finish_gray_stack (nursery_start, nursery_next, GENERATION_NURSERY);
+ TV_GETTIME (atv);
+ time_minor_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
/* walk the pin_queue, build up the fragment list of free memory, unmark
* pinned objects as we go, memzero() the empty fragments so they are ready for the
* next allocations.
*/
build_nursery_fragments (0, next_pin_slot);
- TV_GETTIME (atv);
- DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (btv, atv), fragment_total));
+ TV_GETTIME (btv);
+ time_minor_fragment_creation += TV_ELAPSED_MS (atv, btv);
+ DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %zd bytes available\n", TV_ELAPSED (atv, btv), fragment_total));
- for (section = section_list; section; section = section->block.next) {
- if (section->is_to_space)
- section->is_to_space = FALSE;
- }
+ major_finish_nursery_collection ();
TV_GETTIME (all_btv);
mono_stats.minor_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
g_assert (gray_object_queue_is_empty ());
- commit_stats (GENERATION_NURSERY);
+ check_scan_starts ();
- sections_alloced = num_major_sections - old_num_major_sections;
- minor_collection_sections_alloced += sections_alloced;
+ current_collection_generation = -1;
- return minor_collection_sections_alloced > minor_collection_section_allowance;
+ return major_need_major_collection ();
}
static void
-scan_from_pinned_chunk_if_marked (PinnedChunk *chunk, char *obj, size_t size, void *dummy)
+major_do_collection (const char *reason)
{
- if (object_is_pinned (obj))
- scan_object (obj, NULL, (char*)-1);
-}
-
-static void
-major_collection (const char *reason)
-{
- GCMemSection *section, *prev_section;
LOSObject *bigobj, *prevbo;
- int i;
- PinnedChunk *chunk;
- Fragment *frag;
TV_DECLARE (all_atv);
TV_DECLARE (all_btv);
TV_DECLARE (atv);
*/
char *heap_start = NULL;
char *heap_end = (char*)-1;
- size_t copy_space_required = 0;
int old_num_major_sections = num_major_sections;
int num_major_sections_saved, save_target, allowance_target;
+ //count_ref_nonref_objs ();
+ //consistency_check ();
+
init_stats ();
+ binary_protocol_collection (GENERATION_OLD);
+ check_scan_starts ();
+ gray_object_queue_init ();
degraded_mode = 0;
DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", num_major_gcs));
num_major_gcs++;
mono_stats.major_gc_count ++;
- /* Clear all remaining nursery fragments, pinning depends on this */
- if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) {
- g_assert (nursery_next <= nursery_frag_real_end);
- memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
- for (frag = nursery_fragments; frag; frag = frag->next) {
- memset (frag->fragment_start, 0, frag->fragment_end - frag->fragment_start);
- }
- }
+ /* world must be stopped already */
+ TV_GETTIME (all_atv);
+ TV_GETTIME (atv);
+
+ /* Pinning depends on this */
+ clear_nursery_fragments (nursery_next);
+
+ TV_GETTIME (btv);
+ time_major_pre_collection_fragment_clear += TV_ELAPSED_MS (atv, btv);
if (xdomain_checks)
check_for_xdomain_refs ();
- /*
- * FIXME: implement Mark/Compact
- * Until that is done, we can just apply mostly the same alg as for the nursery:
- * this means we need a big section to potentially copy all the other sections, so
- * it is not ideal specially with large heaps.
- */
- if (g_getenv ("MONO_GC_NO_MAJOR")) {
- collect_nursery (0);
- return;
- }
- TV_GETTIME (all_atv);
- /* FIXME: make sure the nursery next_data ptr is updated */
nursery_section->next_data = nursery_real_end;
/* we should also coalesce scanning from sections close to each other
* and deal with pointers outside of the sections later.
*/
/* The remsets are not useful for a major collection */
clear_remsets ();
- /* world must be stopped already */
+ global_remset_cache_clear ();
+
TV_GETTIME (atv);
init_pinning ();
DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n"));
*/
DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n"));
/* first pass for the sections */
- for (section = section_list; section; section = section->block.next) {
- int start, end;
- DEBUG (6, fprintf (gc_debug_file, "Pinning from section %p (%p-%p)\n", section, section->data, section->end_data));
- find_optimized_pin_queue_area (section->data, section->end_data, &start, &end);
- DEBUG (6, fprintf (gc_debug_file, "Found %d pinning addresses in section %p (%d-%d)\n",
- end - start, section, start, end));
- section->pin_queue_start = start;
- section->pin_queue_end = end;
- }
+ find_section_pin_queue_start_end (nursery_section);
+ major_find_pin_queue_start_ends ();
/* identify possible pointers to the insize of large objects */
DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n"));
for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &start, &end);
if (start != end) {
pin_object (bigobj->data);
+ /* FIXME: only enqueue if object has references */
+ GRAY_OBJECT_ENQUEUE (bigobj->data);
if (heap_dump_file)
pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %zd from roots\n", bigobj->data, safe_name (bigobj->data), bigobj->size));
}
}
- /* look for pinned addresses for pinned-alloc objects */
- DEBUG (6, fprintf (gc_debug_file, "Pinning from pinned-alloc objects\n"));
- for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
- int start, end;
- find_optimized_pin_queue_area (chunk->start_data, (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE, &start, &end);
- if (start != end)
- mark_pinned_from_addresses (chunk, pin_queue + start, pin_queue + end);
- }
/* second pass for the sections */
- for (section = section_list; section; section = section->block.next) {
- int start = section->pin_queue_start;
- int end = section->pin_queue_end;
- if (start != end) {
- int reduced_to;
- reduced_to = pin_objects_from_addresses (section, pin_queue + start, pin_queue + end,
- section->data, section->next_data);
- section->pin_queue_start = start;
- section->pin_queue_end = start + reduced_to;
- }
- copy_space_required += (char*)section->next_data - (char*)section->data;
- }
+ pin_objects_in_section (nursery_section);
+ major_pin_objects ();
TV_GETTIME (btv);
+ time_major_pinning += TV_ELAPSED_MS (atv, btv);
DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", next_pin_slot, TV_ELAPSED (atv, btv)));
DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", next_pin_slot));
- new_to_space_section ();
- gray_object_queue_init ();
+ major_init_to_space ();
+
+ drain_gray_stack ();
+
+ TV_GETTIME (atv);
+ time_major_scan_pinned += TV_ELAPSED_MS (btv, atv);
- /* the old generation doesn't need to be scanned (no remembered sets or card
- * table needed either): the only objects that must survive are those pinned and
- * those referenced by the precise roots.
- * mark any section without pinned objects, so we can free it since we will be able to
- * move all the objects.
- */
- /* the pinned objects are roots (big objects are included in this list, too) */
- for (section = section_list; section; section = section->block.next) {
- for (i = section->pin_queue_start; i < section->pin_queue_end; ++i) {
- DEBUG (6, fprintf (gc_debug_file, "Precise object scan %d of pinned %p (%s)\n",
- i, pin_queue [i], safe_name (pin_queue [i])));
- scan_object (pin_queue [i], heap_start, heap_end);
- }
- }
- for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
- if (object_is_pinned (bigobj->data)) {
- DEBUG (6, fprintf (gc_debug_file, "Precise object scan pinned LOS object %p (%s)\n",
- bigobj->data, safe_name (bigobj->data)));
- scan_object (bigobj->data, heap_start, heap_end);
- }
- }
- scan_pinned_objects (scan_from_pinned_chunk_if_marked, NULL);
/* registered roots, this includes static fields */
- scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_NORMAL);
- scan_from_registered_roots (heap_start, heap_end, ROOT_TYPE_WBARRIER);
+ scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_NORMAL);
+ scan_from_registered_roots (major_copy_or_mark_object, heap_start, heap_end, ROOT_TYPE_WBARRIER);
+ TV_GETTIME (btv);
+ time_major_scan_registered_roots += TV_ELAPSED_MS (atv, btv);
+
/* Threads */
+ /* FIXME: This is the wrong place for this, because it does
+ pinning */
scan_thread_data (heap_start, heap_end, TRUE);
- /* alloc_pinned objects */
- scan_from_pinned_objects (heap_start, heap_end);
+ TV_GETTIME (atv);
+ time_major_scan_thread_data += TV_ELAPSED_MS (btv, atv);
+
+ TV_GETTIME (btv);
+ time_major_scan_alloc_pinned += TV_ELAPSED_MS (atv, btv);
+
/* scan the list of objects ready for finalization */
- scan_finalizer_entries (fin_ready_list, heap_start, heap_end);
- scan_finalizer_entries (critical_fin_list, heap_start, heap_end);
+ scan_finalizer_entries (major_copy_or_mark_object, fin_ready_list);
+ scan_finalizer_entries (major_copy_or_mark_object, critical_fin_list);
TV_GETTIME (atv);
+ time_major_scan_finalized += TV_ELAPSED_MS (btv, atv);
DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv)));
- /* we need to go over the big object list to see if any was marked and scan it
- * And we need to make this in a loop, considering that objects referenced by finalizable
- * objects could reference big objects (this happens in finish_gray_stack ())
- */
- scan_needed_big_objects (heap_start, heap_end);
+ TV_GETTIME (btv);
+ time_major_scan_big_objects += TV_ELAPSED_MS (atv, btv);
+
/* all the objects in the heap */
finish_gray_stack (heap_start, heap_end, GENERATION_OLD);
-
- unset_to_space ();
+ TV_GETTIME (atv);
+ time_major_finish_gray_stack += TV_ELAPSED_MS (btv, atv);
/* sweep the big objects list */
prevbo = NULL;
for (bigobj = los_object_list; bigobj;) {
if (object_is_pinned (bigobj->data)) {
unpin_object (bigobj->data);
- bigobj->scanned = FALSE;
} else {
LOSObject *to_free;
/* not referenced anywhere, so we can free it */
prevbo = bigobj;
bigobj = bigobj->next;
}
- /* unpin objects from the pinned chunks and free the unmarked ones */
- sweep_pinned_objects ();
-
- /* free the unused sections */
- prev_section = NULL;
- for (section = section_list; section;) {
- /* to_space doesn't need handling here and the nursery is special */
- if (section->is_to_space || section == nursery_section) {
- if (section->is_to_space)
- section->is_to_space = FALSE;
- prev_section = section;
- section = section->block.next;
- continue;
- }
- /* no pinning object, so the section is free */
- if (section->pin_queue_start == section->pin_queue_end) {
- GCMemSection *to_free;
- if (prev_section)
- prev_section->block.next = section->block.next;
- else
- section_list = section->block.next;
- to_free = section;
- section = section->block.next;
- free_major_section (to_free);
- continue;
- } else {
- DEBUG (6, fprintf (gc_debug_file, "Section %p has still pinned objects (%d)\n", section, section->pin_queue_end - section->pin_queue_start));
- build_section_fragments (section);
- }
- prev_section = section;
- section = section->block.next;
- }
+
+ major_sweep ();
+
+ TV_GETTIME (btv);
+ time_major_sweep += TV_ELAPSED_MS (atv, btv);
/* walk the pin_queue, build up the fragment list of free memory, unmark
* pinned objects as we go, memzero() the empty fragments so they are ready for the
*/
build_nursery_fragments (nursery_section->pin_queue_start, nursery_section->pin_queue_end);
+ TV_GETTIME (atv);
+ time_major_fragment_creation += TV_ELAPSED_MS (btv, atv);
+
TV_GETTIME (all_btv);
mono_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv);
g_assert (gray_object_queue_is_empty ());
- commit_stats (GENERATION_OLD);
-
num_major_sections_saved = MAX (old_num_major_sections - num_major_sections, 1);
save_target = num_major_sections / 2;
+ /*
+ * We aim to allow the allocation of as many sections as is
+ * necessary to reclaim save_target sections in the next
+ * collection. We assume the collection pattern won't change.
+ * In the last cycle, we had num_major_sections_saved for
+ * minor_collection_sections_alloced. Assuming things won't
+ * change, this must be the same ratio as save_target for
+ * allowance_target, i.e.
+ *
+ * num_major_sections_saved save_target
+ * --------------------------------- == ----------------
+ * minor_collection_sections_alloced allowance_target
+ *
+ * hence:
+ */
allowance_target = save_target * minor_collection_sections_alloced / num_major_sections_saved;
minor_collection_section_allowance = MAX (MIN (allowance_target, num_major_sections), MIN_MINOR_COLLECTION_SECTION_ALLOWANCE);
- /*
- printf ("alloced %d saved %d target %d allowance %d\n",
- minor_collection_sections_alloced, num_major_sections_saved, allowance_target,
- minor_collection_section_allowance);
- */
-
minor_collection_sections_alloced = 0;
-}
-
-/*
- * Allocate a new section of memory to be used as old generation.
- */
-static GCMemSection*
-alloc_major_section (void)
-{
- GCMemSection *section;
- int scan_starts;
-
- section = get_os_memory_aligned (MAJOR_SECTION_SIZE, TRUE);
- section->next_data = section->data = (char*)section + SIZEOF_GC_MEM_SECTION;
- g_assert (!((mword)section->data & 7));
- section->size = MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
- section->end_data = section->data + section->size;
- UPDATE_HEAP_BOUNDARIES (section->data, section->end_data);
- total_alloc += section->size;
- DEBUG (3, fprintf (gc_debug_file, "New major heap section: (%p-%p), total: %zd\n", section->data, section->end_data, total_alloc));
- scan_starts = section->size / SCAN_START_SIZE;
- section->scan_starts = get_internal_mem (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS);
- section->num_scan_start = scan_starts;
- section->block.role = MEMORY_ROLE_GEN1;
- section->is_to_space = TRUE;
-
- /* add to the section list */
- section->block.next = section_list;
- section_list = section;
- ++num_major_sections;
+ check_scan_starts ();
- return section;
+ //consistency_check ();
}
static void
-free_major_section (GCMemSection *section)
+major_collection (const char *reason)
{
- DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
- free_internal_mem (section->scan_starts, INTERNAL_MEM_SCAN_STARTS);
- free_os_memory (section, MAJOR_SECTION_SIZE);
- total_alloc -= MAJOR_SECTION_SIZE - SIZEOF_GC_MEM_SECTION;
+ if (g_getenv ("MONO_GC_NO_MAJOR")) {
+ collect_nursery (0);
+ return;
+ }
- --num_major_sections;
+ current_collection_generation = GENERATION_OLD;
+ major_do_collection (reason);
+ current_collection_generation = -1;
}
/*
report_pinned_chunk (chunk, i++);
}
printf ("Pinned memory usage:\n");
- i = 0;
- for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
- report_pinned_chunk (chunk, i++);
- }
-}
-
-/*
- * the array of pointers from @start to @end contains conservative
- * pointers to objects inside @chunk: mark each referenced object
- * with the PIN bit.
- */
-static void
-mark_pinned_from_addresses (PinnedChunk *chunk, void **start, void **end)
-{
- for (; start < end; start++) {
- char *addr = *start;
- int offset = (char*)addr - (char*)chunk;
- int page = offset / FREELIST_PAGESIZE;
- int obj_offset = page == 0? offset - ((char*)chunk->start_data - (char*)chunk): offset % FREELIST_PAGESIZE;
- int slot_size = chunk->page_sizes [page];
- void **ptr;
- /* the page is not allocated */
- if (!slot_size)
- continue;
- /* would be faster if we restrict the sizes to power of two,
- * but that's a waste of memory: need to measure. it could reduce
- * fragmentation since there are less pages needed, if for example
- * someone interns strings of each size we end up with one page per
- * interned string (still this is just ~40 KB): with more fine-grained sizes
- * this increases the number of used pages.
- */
- if (page == 0) {
- obj_offset /= slot_size;
- obj_offset *= slot_size;
- addr = (char*)chunk->start_data + obj_offset;
- } else {
- obj_offset /= slot_size;
- obj_offset *= slot_size;
- addr = (char*)chunk + page * FREELIST_PAGESIZE + obj_offset;
- }
- ptr = (void**)addr;
- /* if the vtable is inside the chunk it's on the freelist, so skip */
- if (*ptr && (*ptr < (void*)chunk->start_data || *ptr > (void*)((char*)chunk + chunk->num_pages * FREELIST_PAGESIZE))) {
- pin_object (addr);
- if (heap_dump_file)
- pin_stats_register_object ((char*) addr, safe_object_get_size ((MonoObject*) addr));
- DEBUG (6, fprintf (gc_debug_file, "Marked pinned object %p (%s) from roots\n", addr, safe_name (addr)));
- }
- }
-}
-
-static void
-scan_pinned_objects (ScanPinnedObjectCallbackFunc callback, void *callback_data)
-{
- PinnedChunk *chunk;
- int i, obj_size;
- char *p, *endp;
- void **ptr;
- void *end_chunk;
- for (chunk = pinned_chunk_list; chunk; chunk = chunk->block.next) {
- end_chunk = (char*)chunk + chunk->num_pages * FREELIST_PAGESIZE;
- DEBUG (6, fprintf (gc_debug_file, "Scanning pinned chunk %p (range: %p-%p)\n", chunk, chunk->start_data, end_chunk));
- for (i = 0; i < chunk->num_pages; ++i) {
- obj_size = chunk->page_sizes [i];
- if (!obj_size)
- continue;
- p = i? (char*)chunk + i * FREELIST_PAGESIZE: chunk->start_data;
- endp = i? p + FREELIST_PAGESIZE: (char*)chunk + FREELIST_PAGESIZE;
- DEBUG (6, fprintf (gc_debug_file, "Page %d (size: %d, range: %p-%p)\n", i, obj_size, p, endp));
- while (p + obj_size <= endp) {
- ptr = (void**)p;
- DEBUG (9, fprintf (gc_debug_file, "Considering %p (vtable: %p)\n", ptr, *ptr));
- /* if the first word (the vtable) is outside the chunk we have an object */
- if (*ptr && (*ptr < (void*)chunk || *ptr >= end_chunk))
- callback (chunk, (char*)ptr, obj_size, callback_data);
- p += obj_size;
- }
- }
- }
-}
-
-static void
-sweep_pinned_objects_callback (PinnedChunk *chunk, char *ptr, size_t size, void *data)
-{
- if (object_is_pinned (ptr)) {
- unpin_object (ptr);
- DEBUG (6, fprintf (gc_debug_file, "Unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
- } else {
- DEBUG (6, fprintf (gc_debug_file, "Freeing unmarked pinned object %p (%s)\n", ptr, safe_name (ptr)));
- free_pinned_object (chunk, ptr, size);
- }
-}
-
-static void
-sweep_pinned_objects (void)
-{
- scan_pinned_objects (sweep_pinned_objects_callback, NULL);
-}
-
-static void
-scan_object_callback (PinnedChunk *chunk, char *ptr, size_t size, char **data)
-{
- DEBUG (6, fprintf (gc_debug_file, "Precise object scan of alloc_pinned %p (%s)\n", ptr, safe_name (ptr)));
- /* FIXME: Put objects without references into separate chunks
- which do not need to be scanned */
- scan_object (ptr, data [0], data [1]);
-}
-
-static void
-scan_from_pinned_objects (char *addr_start, char *addr_end)
-{
- char *data [2] = { addr_start, addr_end };
- scan_pinned_objects ((ScanPinnedObjectCallbackFunc)scan_object_callback, data);
+ major_report_pinned_memory_usage ();
}
/*
{
PinnedChunk *chunk;
int offset;
- int size = MAJOR_SECTION_SIZE;
+ int size = PINNED_CHUNK_SIZE;
- chunk = get_os_memory_aligned (size, TRUE);
+ chunk = get_os_memory_aligned (size, size, TRUE);
chunk->block.role = MEMORY_ROLE_PINNED;
UPDATE_HEAP_BOUNDARIES (chunk, ((char*)chunk + size));
chunk->page_sizes [0] = PINNED_FIRST_SLOT_SIZE;
build_freelist (chunk, slot_for_size (PINNED_FIRST_SLOT_SIZE), PINNED_FIRST_SLOT_SIZE, chunk->start_data, ((char*)chunk + FREELIST_PAGESIZE));
DEBUG (4, fprintf (gc_debug_file, "Allocated pinned chunk %p, size: %d\n", chunk, size));
- min_pinned_chunk_addr = MIN (min_pinned_chunk_addr, (char*)chunk->start_data);
- max_pinned_chunk_addr = MAX (max_pinned_chunk_addr, ((char*)chunk + size));
return chunk;
}
return NULL;
}
-static void*
-alloc_from_freelist (size_t size)
-{
- int slot;
- void *res = NULL;
- PinnedChunk *pchunk;
- slot = slot_for_size (size);
- /*g_print ("using slot %d for size %d (slot size: %d)\n", slot, size, freelist_sizes [slot]);*/
- g_assert (size <= freelist_sizes [slot]);
- for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
- void **p = pchunk->free_list [slot];
- if (p) {
- /*g_print ("found freelist for slot %d in chunk %p, returning %p, next %p\n", slot, pchunk, p, *p);*/
- pchunk->free_list [slot] = *p;
- return p;
- }
- }
- for (pchunk = pinned_chunk_list; pchunk; pchunk = pchunk->block.next) {
- res = get_chunk_freelist (pchunk, slot);
- if (res)
- return res;
- }
- pchunk = alloc_pinned_chunk ();
- /* FIXME: handle OOM */
- pchunk->block.next = pinned_chunk_list;
- pinned_chunk_list = pchunk;
- res = get_chunk_freelist (pchunk, slot);
- return res;
-}
-
/* used for the GC-internal data structures */
-/* FIXME: add support for bigger sizes by allocating more than one page
- * in the chunk.
- */
static void*
get_internal_mem (size_t size, int type)
{
{
size_t size = obj->size;
DEBUG (4, fprintf (gc_debug_file, "Freed large object %p, size %zd\n", obj->data, obj->size));
+ binary_protocol_empty (obj->data, obj->size);
los_memory_usage -= size;
size += sizeof (LOSObject);
LOSObject *obj;
void **vtslot;
size_t alloc_size;
- int just_did_major_gc = FALSE;
g_assert (size > MAX_SMALL_OBJ_SIZE);
if (los_memory_usage > next_los_collection) {
+ static mword last_los_memory_usage = 0;
+
+ mword los_memory_alloced;
+ mword old_los_memory_usage;
+ mword los_memory_saved;
+ mword save_target;
+ mword allowance_target;
+ mword allowance;
+
DEBUG (4, fprintf (gc_debug_file, "Should trigger major collection: req size %zd (los already: %zu, limit: %zu)\n", size, los_memory_usage, next_los_collection));
- just_did_major_gc = TRUE;
stop_world ();
+
+ g_assert (los_memory_usage >= last_los_memory_usage);
+ los_memory_alloced = los_memory_usage - last_los_memory_usage;
+ old_los_memory_usage = los_memory_usage;
+
major_collection ("LOS overflow");
+
+ los_memory_saved = MAX (old_los_memory_usage - los_memory_usage, 1);
+ save_target = los_memory_usage / 2;
+ /*
+ * see the comment at the end of major_collection()
+ * for the explanation for this calculation.
+ */
+ allowance_target = (mword)((double)save_target * (double)los_memory_alloced / (double)los_memory_saved);
+ allowance = MAX (MIN (allowance_target, los_memory_usage), MIN_LOS_ALLOWANCE);
+ next_los_collection = los_memory_usage + allowance;
+
+ last_los_memory_usage = los_memory_usage;
+
restart_world ();
- /* later increase based on a percent of the heap size */
- next_los_collection = los_memory_usage + 5*1024*1024;
}
alloc_size = size;
alloc_size += sizeof (LOSObject);
alloc_size &= ~(pagesize - 1);
/* FIXME: handle OOM */
obj = get_os_memory (alloc_size, TRUE);
+ g_assert (!((mword)obj->data & (ALLOC_ALIGN - 1)));
obj->size = size;
vtslot = (void**)obj->data;
*vtslot = vtable;
los_memory_usage += size;
los_num_objects++;
DEBUG (4, fprintf (gc_debug_file, "Allocated large object %p, vtable: %p (%s), size: %zd\n", obj->data, vtable, vtable->klass->name, size));
+ binary_protocol_alloc (obj->data, vtable, size);
return obj->data;
}
+static void
+setup_fragment (Fragment *frag, Fragment *prev, size_t size)
+{
+ /* remove from the list */
+ if (prev)
+ prev->next = frag->next;
+ else
+ nursery_fragments = frag->next;
+ nursery_next = frag->fragment_start;
+ nursery_frag_real_end = frag->fragment_end;
+
+ DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
+ frag->next = fragment_freelist;
+ fragment_freelist = frag;
+}
+
/* check if we have a suitable fragment in nursery_fragments to be able to allocate
* an object of size @size
* Return FALSE if not found (which means we need a collection)
prev = NULL;
for (frag = nursery_fragments; frag; frag = frag->next) {
if (size <= (frag->fragment_end - frag->fragment_start)) {
- /* remove from the list */
- if (prev)
- prev->next = frag->next;
- else
- nursery_fragments = frag->next;
- nursery_next = frag->fragment_start;
- nursery_frag_real_end = frag->fragment_end;
-
- DEBUG (4, fprintf (gc_debug_file, "Using nursery fragment %p-%p, size: %zd (req: %zd)\n", nursery_next, nursery_frag_real_end, nursery_frag_real_end - nursery_next, size));
- frag->next = fragment_freelist;
- fragment_freelist = frag;
+ setup_fragment (frag, prev, size);
return TRUE;
}
prev = frag;
}
/*
- * size is already rounded up and we hold the GC lock.
+ * Same as search_fragment_for_size but if search for @desired_size fails, try to satisfy @minimum_size.
+ * This improves nursery usage.
*/
-static void*
-alloc_degraded (MonoVTable *vtable, size_t size)
+static int
+search_fragment_for_size_range (size_t desired_size, size_t minimum_size)
{
- GCMemSection *section;
- void **p = NULL;
- g_assert (size <= MAX_SMALL_OBJ_SIZE);
- for (section = section_list; section; section = section->block.next) {
- if (section != nursery_section && (section->end_data - section->next_data) >= size) {
- p = (void**)section->next_data;
- break;
+ Fragment *frag, *prev, *min_prev;
+ DEBUG (4, fprintf (gc_debug_file, "Searching nursery fragment %p, desired size: %zd minimum size %zd\n", nursery_frag_real_end, desired_size, minimum_size));
+
+ if (nursery_frag_real_end > nursery_next && nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
+ /* Clear the remaining space, pinning depends on this */
+ memset (nursery_next, 0, nursery_frag_real_end - nursery_next);
+
+ min_prev = GINT_TO_POINTER (-1);
+ prev = NULL;
+
+ for (frag = nursery_fragments; frag; frag = frag->next) {
+ int frag_size = frag->fragment_end - frag->fragment_start;
+ if (desired_size <= frag_size) {
+ setup_fragment (frag, prev, desired_size);
+ return desired_size;
}
+ if (minimum_size <= frag_size)
+ min_prev = prev;
+
+ prev = frag;
}
- if (!p) {
- section = alloc_major_section ();
- section->is_to_space = FALSE;
- /* FIXME: handle OOM */
- p = (void**)section->next_data;
+
+ if (min_prev != GINT_TO_POINTER (-1)) {
+ int frag_size;
+ if (min_prev)
+ frag = min_prev->next;
+ else
+ frag = nursery_fragments;
+
+ frag_size = frag->fragment_end - frag->fragment_start;
+ HEAVY_STAT (++stat_wasted_fragments_used);
+ HEAVY_STAT (stat_wasted_fragments_bytes += frag_size);
+
+ setup_fragment (frag, min_prev, minimum_size);
+ return frag_size;
}
- section->next_data += size;
- degraded_mode += size;
- DEBUG (3, fprintf (gc_debug_file, "Allocated (degraded) object %p, vtable: %p (%s), size: %zd in section %p\n", p, vtable, vtable->klass->name, size, section));
- *p = vtable;
- return p;
+
+ return 0;
}
/*
/* FIXME: handle OOM */
void **p;
char *new_next;
- gboolean res;
TLAB_ACCESS_INIT;
HEAVY_STAT (++stat_objects_alloced);
+ if (size <= MAX_SMALL_OBJ_SIZE)
+ HEAVY_STAT (stat_bytes_alloced += size);
+ else
+ HEAVY_STAT (stat_bytes_alloced_los += size);
size += ALLOC_ALIGN - 1;
size &= ~(ALLOC_ALIGN - 1);
*/
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
+ binary_protocol_alloc (p , vtable, size);
g_assert (*p == NULL);
*p = vtable;
return p;
}
+ /*FIXME This codepath is current deadcode since tlab_size > MAX_SMALL_OBJ_SIZE*/
if (size > tlab_size) {
/* Allocate directly from the nursery */
if (nursery_next + size >= nursery_frag_real_end) {
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
memset (p, 0, size);
} else {
+ int alloc_size = tlab_size;
+ int available_in_nursery = nursery_frag_real_end - nursery_next;
if (TLAB_START)
DEBUG (3, fprintf (gc_debug_file, "Retire TLAB: %p-%p [%ld]\n", TLAB_START, TLAB_REAL_END, (long)(TLAB_REAL_END - TLAB_NEXT - size)));
- if (nursery_next + tlab_size >= nursery_frag_real_end) {
- res = search_fragment_for_size (tlab_size);
- if (!res) {
- minor_collect_or_expand_inner (tlab_size);
- if (degraded_mode) {
- p = alloc_degraded (vtable, size);
- return p;
+ if (alloc_size >= available_in_nursery) {
+ if (available_in_nursery > MAX_NURSERY_TLAB_WASTE && available_in_nursery > size) {
+ alloc_size = available_in_nursery;
+ } else {
+ alloc_size = search_fragment_for_size_range (tlab_size, size);
+ if (!alloc_size) {
+ alloc_size = tlab_size;
+ minor_collect_or_expand_inner (tlab_size);
+ if (degraded_mode) {
+ p = alloc_degraded (vtable, size);
+ return p;
+ }
}
}
}
/* Allocate a new TLAB from the current nursery fragment */
TLAB_START = nursery_next;
- nursery_next += tlab_size;
+ nursery_next += alloc_size;
TLAB_NEXT = TLAB_START;
- TLAB_REAL_END = TLAB_START + tlab_size;
- TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, tlab_size);
+ TLAB_REAL_END = TLAB_START + alloc_size;
+ TLAB_TEMP_END = TLAB_START + MIN (SCAN_START_SIZE, alloc_size);
if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION)
- memset (TLAB_START, 0, tlab_size);
+ memset (TLAB_START, 0, alloc_size);
/* Allocate from the TLAB */
p = (void*)TLAB_NEXT;
}
DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
+ binary_protocol_alloc (p, vtable, size);
*p = vtable;
return p;
}
+static void*
+mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size)
+{
+ void **p;
+ char *new_next;
+ TLAB_ACCESS_INIT;
+
+ size += ALLOC_ALIGN - 1;
+ size &= ~(ALLOC_ALIGN - 1);
+
+ g_assert (vtable->gc_descr);
+ if (size <= MAX_SMALL_OBJ_SIZE) {
+ /* tlab_next and tlab_temp_end are TLS vars so accessing them might be expensive */
+
+ p = (void**)TLAB_NEXT;
+ /* FIXME: handle overflow */
+ new_next = (char*)p + size;
+ TLAB_NEXT = new_next;
+
+ if (G_LIKELY (new_next < TLAB_TEMP_END)) {
+ /* Fast path */
+
+ /*
+ * FIXME: We might need a memory barrier here so the change to tlab_next is
+ * visible before the vtable store.
+ */
+
+ HEAVY_STAT (++stat_objects_alloced);
+ HEAVY_STAT (stat_bytes_alloced += size);
+
+ DEBUG (6, fprintf (gc_debug_file, "Allocated object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
+ binary_protocol_alloc (p, vtable, size);
+ g_assert (*p == NULL);
+ *p = vtable;
+
+ g_assert (TLAB_NEXT == new_next);
+
+ return p;
+ }
+ }
+ return NULL;
+}
+
void*
mono_gc_alloc_obj (MonoVTable *vtable, size_t size)
{
void *res;
+#ifndef DISABLE_CRITICAL_REGION
+ TLAB_ACCESS_INIT;
+ ENTER_CRITICAL_REGION;
+ res = mono_gc_try_alloc_obj_nolock (vtable, size);
+ if (res) {
+ EXIT_CRITICAL_REGION;
+ return res;
+ }
+ EXIT_CRITICAL_REGION;
+#endif
LOCK_GC;
res = mono_gc_alloc_obj_nolock (vtable, size);
UNLOCK_GC;
}
void*
-mono_gc_alloc_vector (MonoVTable *vtable, size_t size, mono_array_size_t max_length)
+mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length)
{
MonoArray *arr;
+#ifndef DISABLE_CRITICAL_REGION
+ TLAB_ACCESS_INIT;
+ ENTER_CRITICAL_REGION;
+ arr = mono_gc_try_alloc_obj_nolock (vtable, size);
+ if (arr) {
+ arr->max_length = max_length;
+ EXIT_CRITICAL_REGION;
+ return arr;
+ }
+ EXIT_CRITICAL_REGION;
+#endif
LOCK_GC;
}
void*
-mono_gc_alloc_array (MonoVTable *vtable, size_t size, mono_array_size_t max_length, mono_array_size_t bounds_size)
+mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uintptr_t bounds_size)
{
MonoArray *arr;
MonoArrayBounds *bounds;
mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len)
{
MonoString *str;
+#ifndef DISABLE_CRITICAL_REGION
+ TLAB_ACCESS_INIT;
+ ENTER_CRITICAL_REGION;
+ str = mono_gc_try_alloc_obj_nolock (vtable, size);
+ if (str) {
+ str->length = len;
+ EXIT_CRITICAL_REGION;
+ return str;
+ }
+ EXIT_CRITICAL_REGION;
+#endif
LOCK_GC;
size += ALLOC_ALIGN - 1;
size &= ~(ALLOC_ALIGN - 1);
LOCK_GC;
- if (size > MAX_FREELIST_SIZE) {
+ if (size > MAX_SMALL_OBJ_SIZE) {
/* large objects are always pinned anyway */
p = alloc_large_inner (vtable, size);
} else {
- p = alloc_from_freelist (size);
- memset (p, 0, size);
+ DEBUG (9, g_assert (vtable->klass->inited));
+ p = major_alloc_small_pinned_obj (size, vtable->klass->has_references);
}
DEBUG (6, fprintf (gc_debug_file, "Allocated pinned object %p, vtable: %p (%s), size: %zd\n", p, vtable, vtable->klass->name, size));
+ binary_protocol_alloc (p, vtable, size);
*p = vtable;
UNLOCK_GC;
return p;
/* LOCKING: requires that the GC lock is held */
static void
-finalize_in_range (char *start, char *end, int generation)
+finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
{
FinalizeEntryHashTable *hash_table = get_finalize_entry_hash_table (generation);
FinalizeEntry *entry, *prev;
for (i = 0; i < finalizable_hash_size; ++i) {
prev = NULL;
for (entry = finalizable_hash [i]; entry;) {
- if ((char*)entry->object >= start && (char*)entry->object < end && !object_is_in_to_space (entry->object)) {
+ if ((char*)entry->object >= start && (char*)entry->object < end && !major_is_object_live (entry->object)) {
gboolean is_fin_ready = object_is_fin_ready (entry->object);
- char *copy = copy_object (entry->object, start, end);
+ char *copy = entry->object;
+ copy_func ((void**)©);
if (is_fin_ready) {
char *from;
FinalizeEntry *next;
finalizable_hash [i] = entry->next;
hash_table->num_registered--;
- entry->object = copy;
+ entry->object = copy;
+
+ /* insert it into the major hash */
+ rehash_fin_table_if_necessary (&major_finalizable_hash);
+ major_hash = mono_object_hash ((MonoObject*) copy) %
+ major_finalizable_hash.size;
+ entry->next = major_finalizable_hash.table [major_hash];
+ major_finalizable_hash.table [major_hash] = entry;
+ major_finalizable_hash.num_registered++;
+
+ DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
+
+ entry = next;
+ continue;
+ } else {
+ /* update pointer */
+ DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
+ entry->object = copy;
+ }
+ }
+ }
+ prev = entry;
+ entry = entry->next;
+ }
+ }
+}
+
+static int
+object_is_reachable (char *object, char *start, char *end)
+{
+ /*This happens for non nursery objects during minor collections. We just treat all objects as alive.*/
+ if (object < start || object >= end)
+ return TRUE;
+ return !object_is_fin_ready (object) || major_is_object_live (object);
+}
+
+/* LOCKING: requires that the GC lock is held */
+static void
+null_ephemerons_for_domain (MonoDomain *domain)
+{
+ EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
+
+ while (current) {
+ MonoObject *object = (MonoObject*)current->array;
+
+ if (object && !object->vtable) {
+ EphemeronLinkNode *tmp = current;
+
+ if (prev)
+ prev->next = current->next;
+ else
+ ephemeron_list = current->next;
+
+ current = current->next;
+ free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
+ } else {
+ prev = current;
+ current = current->next;
+ }
+ }
+}
+
+/* LOCKING: requires that the GC lock is held */
+static void
+clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char *end)
+{
+ int was_in_nursery, was_promoted;
+ EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
+ MonoArray *array;
+ Ephemeron *cur, *array_end;
+ char *tombstone;
+
+ while (current) {
+ char *object = current->array;
+
+ if (!object_is_reachable (object, start, end)) {
+ EphemeronLinkNode *tmp = current;
+
+ DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object));
+
+ if (prev)
+ prev->next = current->next;
+ else
+ ephemeron_list = current->next;
+
+ current = current->next;
+ free_internal_mem (tmp, INTERNAL_MEM_EPHEMERON_LINK);
+
+ continue;
+ }
+
+ was_in_nursery = ptr_in_nursery (object);
+ copy_func ((void**)&object);
+ current->array = object;
+
+ /*The array was promoted, add global remsets for key/values left behind in nursery.*/
+ was_promoted = was_in_nursery && !ptr_in_nursery (object);
- /* insert it into the major hash */
- rehash_fin_table_if_necessary (&major_finalizable_hash);
- major_hash = mono_object_hash ((MonoObject*) copy) %
- major_finalizable_hash.size;
- entry->next = major_finalizable_hash.table [major_hash];
- major_finalizable_hash.table [major_hash] = entry;
- major_finalizable_hash.num_registered++;
+ DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object));
- DEBUG (5, fprintf (gc_debug_file, "Promoting finalization of object %p (%s) (was at %p) to major table\n", copy, safe_name (copy), from));
+ array = (MonoArray*)object;
+ cur = mono_array_addr (array, Ephemeron, 0);
+ array_end = cur + mono_array_length_fast (array);
+ tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
- entry = next;
- continue;
- } else {
- /* update pointer */
- DEBUG (5, fprintf (gc_debug_file, "Updating object for finalization: %p (%s) (was at %p)\n", entry->object, safe_name (entry->object), from));
- entry->object = copy;
- }
+ for (; cur < array_end; ++cur) {
+ char *key = (char*)cur->key;
+
+ if (!key || key == tombstone)
+ continue;
+
+ DEBUG (5, fprintf (gc_debug_file, "[%d] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
+ key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
+ cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
+
+ if (!object_is_reachable (key, start, end)) {
+ cur->key = tombstone;
+ cur->value = NULL;
+ continue;
+ }
+
+ if (was_promoted) {
+ if (ptr_in_nursery (key)) {/*key was not promoted*/
+ DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key));
+ add_to_global_remset (&cur->key);
+ }
+ if (ptr_in_nursery (cur->value)) {/*value was not promoted*/
+ DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value));
+ add_to_global_remset (&cur->value);
+ }
+ }
+ }
+ prev = current;
+ current = current->next;
+ }
+}
+
+/* LOCKING: requires that the GC lock is held */
+static int
+mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end)
+{
+ int nothing_marked = 1;
+ EphemeronLinkNode *current = ephemeron_list;
+ MonoArray *array;
+ Ephemeron *cur, *array_end;
+ char *tombstone;
+
+ for (current = ephemeron_list; current; current = current->next) {
+ char *object = current->array;
+ DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object));
+
+ /*We ignore arrays in old gen during minor collections since all objects are promoted by the remset machinery.*/
+ if (object < start || object >= end)
+ continue;
+
+ /*It has to be alive*/
+ if (!object_is_reachable (object, start, end)) {
+ DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n"));
+ continue;
+ }
+
+ copy_func ((void**)&object);
+
+ array = (MonoArray*)object;
+ cur = mono_array_addr (array, Ephemeron, 0);
+ array_end = cur + mono_array_length_fast (array);
+ tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
+
+ for (; cur < array_end; ++cur) {
+ char *key = cur->key;
+
+ if (!key || key == tombstone)
+ continue;
+
+ DEBUG (5, fprintf (gc_debug_file, "[%d] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0),
+ key, object_is_reachable (key, start, end) ? "reachable" : "unreachable",
+ cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"));
+
+ if (object_is_reachable (key, start, end)) {
+ char *value = cur->value;
+
+ copy_func ((void**)&cur->key);
+ if (value) {
+ if (!object_is_reachable (value, start, end))
+ nothing_marked = 0;
+ copy_func ((void**)&cur->value);
}
}
- prev = entry;
- entry = entry->next;
}
}
+
+ DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked));
+ return nothing_marked;
}
/* LOCKING: requires that the GC lock is held */
static void
-null_link_in_range (char *start, char *end, int generation)
+null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation)
{
DisappearingLinkHashTable *hash = get_dislink_hash_table (generation);
DisappearingLink **disappearing_link_hash = hash->table;
prev = NULL;
for (entry = disappearing_link_hash [i]; entry;) {
char *object = DISLINK_OBJECT (entry);
- if (object >= start && object < end && !object_is_in_to_space (object)) {
+ if (object >= start && object < end && !major_is_object_live (object)) {
gboolean track = DISLINK_TRACK (entry);
if (!track && object_is_fin_ready (object)) {
void **p = entry->link;
hash->num_links--;
continue;
} else {
- char *copy = copy_object (object, start, end);
+ char *copy = object;
+ copy_func ((void**)©);
/* Update pointer if it's moved. If the object
* has been moved out of the nursery, we need to
prev = NULL;
for (entry = disappearing_link_hash [i]; entry; ) {
char *object = DISLINK_OBJECT (entry);
- /* FIXME: actually there should be no object
- left in the domain with a non-null vtable
- (provided we remove the Thread special
- case) */
- if (object && (!((MonoObject*)object)->vtable || mono_object_domain (object) == domain)) {
+ if (object && !((MonoObject*)object)->vtable) {
DisappearingLink *next = entry->next;
if (prev)
mono_gc_invoke_finalizers (void)
{
FinalizeEntry *entry = NULL;
- gboolean entry_is_critical;
+ gboolean entry_is_critical = FALSE;
int count = 0;
void *obj;
/* FIXME: batch to reduce lock contention */
{
void *ptr = cur_thread_regs;
SgenThreadInfo *info = thread_info_lookup (ARCH_GET_THREAD ());
+
info->stack_start = align_pointer (&ptr);
g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end);
ARCH_STORE_REGS (ptr);
*/
//#define XDOMAIN_CHECKS_IN_WBARRIER
+#ifndef BINARY_PROTOCOL
#ifndef HEAVY_STATISTICS
#define MANAGED_ALLOCATION
#ifndef XDOMAIN_CHECKS_IN_WBARRIER
#define MANAGED_WBARRIER
#endif
#endif
+#endif
static gboolean
is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip);
for (info = thread_table [i]; info; info = info->next) {
if (info->skip)
continue;
- if (!info->stack_start ||
+ if (!info->stack_start || info->in_critical_region ||
is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip)) {
+ binary_protocol_thread_restart ((gpointer)info->id);
result = pthread_kill (info->id, restart_signal_num);
if (result == 0) {
++restart_count;
if (gc_callbacks.thread_suspend_func)
gc_callbacks.thread_suspend_func (info->runtime_data, context);
+ DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for suspend from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
/* notify the waiting thread */
MONO_SEM_POST (suspend_ack_semaphore_ptr);
info->stop_count = stop_count;
sigsuspend (&suspend_signal_mask);
} while (info->signal != restart_signal_num);
+ DEBUG (4, fprintf (gc_debug_file, "Posting suspend_ack_semaphore for resume from %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
/* notify the waiting thread */
MONO_SEM_POST (suspend_ack_semaphore_ptr);
info = thread_info_lookup (pthread_self ());
info->signal = restart_signal_num;
+ DEBUG (4, fprintf (gc_debug_file, "Restart handler in %p %p\n", info, (gpointer)ARCH_GET_THREAD ()));
errno = old_errno;
}
+static void
+acquire_gc_locks (void)
+{
+ LOCK_INTERRUPTION;
+}
+
+static void
+release_gc_locks (void)
+{
+ UNLOCK_INTERRUPTION;
+}
+
static TV_DECLARE (stop_world_time);
static unsigned long max_pause_usec = 0;
{
int count;
+ acquire_gc_locks ();
+
update_current_thread_stack (&count);
global_stop_count++;
TV_DECLARE (end_sw);
unsigned long usec;
+ /* notify the profiler of the leftovers */
+ if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) {
+ if (moved_objects_idx) {
+ mono_profiler_gc_moves (moved_objects, moved_objects_idx);
+ moved_objects_idx = 0;
+ }
+ }
for (i = 0; i < THREAD_HASH_SIZE; ++i) {
for (info = thread_table [i]; info; info = info->next) {
info->stack_start = NULL;
}
}
+ release_gc_locks ();
+
count = thread_handshake (restart_signal_num);
TV_GETTIME (end_sw);
usec = TV_ELAPSED (stop_world_time, end_sw);
void*
mono_gc_scan_object (void *obj)
{
- return copy_object (obj, scan_area_arg_start, scan_area_arg_end);
+ if (current_collection_generation == GENERATION_NURSERY)
+ copy_object (&obj);
+ else
+ major_copy_or_mark_object (&obj);
+ return obj;
}
-
+
/*
* Mark from thread stacks and registers.
*/
for (i = 0; i < THREAD_HASH_SIZE; ++i) {
for (info = thread_table [i]; info; info = info->next) {
if (info->skip) {
- DEBUG (2, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
+ DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %zd\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start));
continue;
}
DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %zd, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, next_pin_slot));
ptr = (void**)(*p);
//__builtin_prefetch (ptr);
if (((void*)ptr < start_nursery || (void*)ptr >= end_nursery)) {
- *ptr = copy_object (*ptr, start_nursery, end_nursery);
+ gpointer old = *ptr;
+ copy_object (ptr);
DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p\n", ptr, *ptr));
+ if (old)
+ binary_protocol_ptr_update (ptr, old, *ptr, (gpointer)LOAD_VTABLE (*ptr), safe_object_get_size (*ptr));
if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
/*
* If the object is pinned, each reference to it from nonpinned objects
* becomes part of the global remset, which can grow very large.
*/
DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
- add_to_global_remset (ptr, FALSE);
+ add_to_global_remset (ptr);
}
} else {
DEBUG (9, fprintf (gc_debug_file, "Skipping remset at %p holding %p\n", ptr, *ptr));
return p + 2;
count = p [1];
while (count-- > 0) {
- *ptr = copy_object (*ptr, start_nursery, end_nursery);
+ copy_object (ptr);
DEBUG (9, fprintf (gc_debug_file, "Overwrote remset at %p with %p (count: %d)\n", ptr, *ptr, (int)count));
if (!global && *ptr >= start_nursery && *ptr < end_nursery)
- add_to_global_remset (ptr, FALSE);
+ add_to_global_remset (ptr);
++ptr;
}
return p + 2;
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
return p + 1;
- scan_object ((char*)ptr, start_nursery, end_nursery);
+ scan_object ((char*)ptr);
return p + 1;
- case REMSET_OTHER: {
+ case REMSET_VTYPE: {
ptr = (void**)(*p & ~REMSET_TYPE_MASK);
-
- switch (p [1]) {
- case REMSET_VTYPE:
- if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
- return p + 4;
- desc = p [2];
- count = p [3];
- while (count-- > 0)
- ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
- return p + 4;
- case REMSET_ROOT_LOCATION:
- /* Same as REMSET_LOCATION, but the address is not required to be in the heap */
- *ptr = copy_object (*ptr, start_nursery, end_nursery);
- DEBUG (9, fprintf (gc_debug_file, "Overwrote root location remset at %p with %p\n", ptr, *ptr));
- if (!global && *ptr >= start_nursery && *ptr < end_nursery) {
- /*
- * If the object is pinned, each reference to it from nonpinned objects
- * becomes part of the global remset, which can grow very large.
- */
- DEBUG (9, fprintf (gc_debug_file, "Add to global remset because of pinning %p (%p %s)\n", ptr, *ptr, safe_name (*ptr)));
- add_to_global_remset (ptr, TRUE);
- }
- return p + 2;
- default:
- g_assert_not_reached ();
- }
- break;
+ if (((void*)ptr >= start_nursery && (void*)ptr < end_nursery))
+ return p + 3;
+ desc = p [1];
+ count = p [2];
+ while (count-- > 0)
+ ptr = (void**) scan_vtype ((char*)ptr, desc, start_nursery, end_nursery);
+ return p + 3;
}
default:
g_assert_not_reached ();
case REMSET_OBJECT:
p += 1;
break;
- case REMSET_OTHER:
- switch (p [1]) {
- case REMSET_VTYPE:
- p += 4;
- break;
- case REMSET_ROOT_LOCATION:
- p += 2;
- break;
- default:
- g_assert_not_reached ();
- }
+ case REMSET_VTYPE:
+ p += 3;
break;
default:
g_assert_not_reached ();
DEBUG (4, fprintf (gc_debug_file, "Scanning global remset range: %p-%p, size: %zd\n", remset->data, remset->store_next, remset->store_next - remset->data));
store_pos = remset->data;
for (p = remset->data; p < remset->store_next; p = next_p) {
- mword ptr;
+ void **ptr = p [0];
+
+ /*Ignore previously processed remset.*/
+ if (!global_remset_location_was_not_added (ptr)) {
+ next_p = p + 1;
+ continue;
+ }
next_p = handle_remset (p, start_nursery, end_nursery, TRUE);
* Clear global remsets of locations which no longer point to the
* nursery. Otherwise, they could grow indefinitely between major
* collections.
+ *
+ * Since all global remsets are location remsets, we don't need to unmask the pointer.
*/
- ptr = (p [0] & ~REMSET_TYPE_MASK);
- if ((p [0] & REMSET_TYPE_MASK) == REMSET_LOCATION) {
- if (ptr_in_nursery (*(void**)ptr))
- *store_pos ++ = p [0];
- } else {
- g_assert ((p [0] & REMSET_TYPE_MASK) == REMSET_OTHER);
- g_assert (p [1] == REMSET_ROOT_LOCATION);
- if (ptr_in_nursery (*(void**)ptr)) {
- *store_pos ++ = p [0];
- *store_pos ++ = p [1];
- }
+ if (ptr_in_nursery (*ptr)) {
+ *store_pos ++ = p [0];
+ HEAVY_STAT (++stat_global_remsets_readded);
}
}
next = remset->next;
remset->next = NULL;
if (remset != info->remset) {
- DEBUG (1, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
+ DEBUG (3, fprintf (gc_debug_file, "Freed remset at %p\n", remset->data));
free_internal_mem (remset, INTERNAL_MEM_REMSET);
}
}
if (!info)
return NULL;
+ memset (info, 0, sizeof (SgenThreadInfo));
#ifndef HAVE_KW_THREAD
info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
g_assert (!pthread_getspecific (thread_info_key));
pthread_setspecific (thread_info_key, info);
+#else
+ thread_info = info;
#endif
info->id = ARCH_GET_THREAD ();
info->stopped_domain = NULL;
info->stopped_regs = NULL;
+ binary_protocol_thread_register ((gpointer)info->id);
+
#ifdef HAVE_KW_THREAD
tlab_next_addr = &tlab_next;
store_remset_buffer_index_addr = &store_remset_buffer_index;
}
#elif defined(HAVE_PTHREAD_GET_STACKSIZE_NP) && defined(HAVE_PTHREAD_GET_STACKADDR_NP)
info->stack_end = (char*)pthread_get_stackaddr_np (pthread_self ());
+ info->stack_start_limit = (char*)info->stack_end - pthread_get_stacksize_np (pthread_self ());
#else
{
/* FIXME: we assume the stack grows down */
RememberedSet *rset;
ARCH_THREAD_TYPE id = ARCH_GET_THREAD ();
+ binary_protocol_thread_unregister ((gpointer)id);
+
hash = HASH_PTHREAD_T (id) % THREAD_HASH_SIZE;
p = thread_table [hash];
assert (p);
* Note: the write barriers first do the needed GC work and then do the actual store:
* this way the value is visible to the conservative GC scan after the write barrier
* itself. If a GC interrupts the barrier in the middle, value will be kept alive by
- * the conservative scan, otherwise by the remembered set scan. FIXME: figure out what
- * happens when we need to record which pointers contain references to the new generation.
- * The write barrier will be executed, but the pointer is still not stored.
+ * the conservative scan, otherwise by the remembered set scan.
*/
void
mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
static char *found_obj;
static void
-find_object_for_ptr_in_pinned_chunk_callback (PinnedChunk *chunk, char *obj, size_t size, char *ptr)
+find_object_for_ptr_callback (char *obj, size_t size, char *ptr)
{
if (ptr >= obj && ptr < obj + size) {
g_assert (!found_obj);
char*
find_object_for_ptr (char *ptr)
{
- GCMemSection *section;
LOSObject *bigobj;
- for (section = section_list; section; section = section->block.next) {
- if (ptr >= section->data && ptr < section->end_data)
- return find_object_for_ptr_in_area (ptr, section->data, section->end_data);
- }
+ if (ptr >= nursery_section->data && ptr < nursery_section->end_data)
+ return find_object_for_ptr_in_area (ptr, nursery_section->data, nursery_section->end_data);
for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
if (ptr >= bigobj->data && ptr < bigobj->data + bigobj->size)
return bigobj->data;
}
+ /*
+ * Very inefficient, but this is debugging code, supposed to
+ * be called from gdb, so we don't care.
+ */
found_obj = NULL;
- scan_pinned_objects ((ScanPinnedObjectCallbackFunc)find_object_for_ptr_in_pinned_chunk_callback, ptr);
+ major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)find_object_for_ptr_callback, ptr);
return found_obj;
}
#endif
LOCK_GC;
+
+ if (*(gpointer*)ptr)
+ binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr));
+
if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) {
DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr));
UNLOCK_GC;
mono_gc_wbarrier_generic_nostore (ptr);
}
-void
-mono_gc_wbarrier_set_root (gpointer ptr, MonoObject *value)
-{
- RememberedSet *rs;
- TLAB_ACCESS_INIT;
- HEAVY_STAT (++stat_wbarrier_set_root);
- if (ptr_in_nursery (ptr))
- return;
- DEBUG (8, fprintf (gc_debug_file, "Adding root remset at %p (%s)\n", ptr, value ? safe_name (value) : "null"));
-
- rs = REMEMBERED_SET;
- if (rs->store_next + 2 < rs->end_set) {
- *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
- *(void**)ptr = value;
- return;
- }
- rs = alloc_remset (rs->end_set - rs->data, (void*)1);
- rs->next = REMEMBERED_SET;
- REMEMBERED_SET = rs;
-#ifdef HAVE_KW_THREAD
- thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
-#endif
- *(rs->store_next++) = (mword)ptr | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_ROOT_LOCATION;
-
- *(void**)ptr = value;
-}
-
void
mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
{
DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass));
if (rs->store_next + 3 < rs->end_set) {
- *(rs->store_next++) = (mword)dest | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_VTYPE;
+ *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
*(rs->store_next++) = (mword)klass->gc_descr;
*(rs->store_next++) = (mword)count;
UNLOCK_GC;
#ifdef HAVE_KW_THREAD
thread_info_lookup (ARCH_GET_THREAD ())->remset = rs;
#endif
- *(rs->store_next++) = (mword)dest | REMSET_OTHER;
- *(rs->store_next++) = (mword)REMSET_VTYPE;
+ *(rs->store_next++) = (mword)dest | REMSET_VTYPE;
*(rs->store_next++) = (mword)klass->gc_descr;
*(rs->store_next++) = (mword)count;
UNLOCK_GC;
TLAB_ACCESS_INIT;
HEAVY_STAT (++stat_wbarrier_object_copy);
rs = REMEMBERED_SET;
- DEBUG (1, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
+ DEBUG (6, fprintf (gc_debug_file, "Adding object remset for %p\n", obj));
size = mono_object_class (obj)->instance_size;
LOCK_GC;
/* do not copy the sync state */
void
describe_ptr (char *ptr)
{
- GCMemSection *section;
MonoVTable *vtable;
mword desc;
int type;
if (ptr_in_nursery (ptr)) {
printf ("Pointer inside nursery.\n");
} else {
- for (section = section_list; section;) {
- if (ptr >= section->data && ptr < section->data + section->size)
- break;
- section = section->block.next;
- }
-
- if (section) {
+ if (major_ptr_is_in_non_pinned_space (ptr)) {
printf ("Pointer inside oldspace.\n");
} else if (obj_is_from_pinned_alloc (ptr)) {
printf ("Pointer is inside a pinned chunk.\n");
if ((void**)addr >= ptr && (void**)addr < ptr + count)
*found = TRUE;
return p + 1;
- case REMSET_OTHER: {
- switch (p [1]) {
- case REMSET_VTYPE:
- ptr = (void**)(*p & ~REMSET_TYPE_MASK);
- desc = p [2];
- count = p [3];
-
- switch (desc & 0x7) {
- case DESC_TYPE_RUN_LENGTH:
- OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
- /* The descriptor includes the size of MonoObject */
- skip_size -= sizeof (MonoObject);
- skip_size *= count;
- if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
- *found = TRUE;
- break;
- default:
- // FIXME:
- g_assert_not_reached ();
- }
+ case REMSET_VTYPE:
+ ptr = (void**)(*p & ~REMSET_TYPE_MASK);
+ desc = p [1];
+ count = p [2];
- return p + 4;
- case REMSET_ROOT_LOCATION:
- return p + 2;
+ switch (desc & 0x7) {
+ case DESC_TYPE_RUN_LENGTH:
+ OBJ_RUN_LEN_SIZE (skip_size, desc, ptr);
+ break;
+ case DESC_TYPE_SMALL_BITMAP:
+ OBJ_BITMAP_SIZE (skip_size, desc, start);
+ break;
default:
+ // FIXME:
g_assert_not_reached ();
}
- break;
- }
+
+ /* The descriptor includes the size of MonoObject */
+ skip_size -= sizeof (MonoObject);
+ skip_size *= count;
+ if ((void**)addr >= ptr && (void**)addr < ptr + (skip_size / sizeof (gpointer)))
+ *found = TRUE;
+
+ return p + 3;
default:
g_assert_not_reached ();
}
return FALSE;
}
+static gboolean missing_remsets;
+
+/*
+ * We let a missing remset slide if the target object is pinned,
+ * because the store might have happened but the remset not yet added,
+ * but in that case the target must be pinned. We might theoretically
+ * miss some missing remsets this way, but it's very unlikely.
+ */
#undef HANDLE_PTR
#define HANDLE_PTR(ptr,obj) do { \
if (*(ptr) && (char*)*(ptr) >= nursery_start && (char*)*(ptr) < nursery_next) { \
- if (!find_in_remsets ((char*)(ptr))) { \
+ if (!find_in_remsets ((char*)(ptr))) { \
fprintf (gc_debug_file, "Oldspace->newspace reference %p at offset %zd in object %p (%s.%s) not found in remsets.\n", *(ptr), (char*)(ptr) - (char*)(obj), (obj), ((MonoObject*)(obj))->vtable->klass->name_space, ((MonoObject*)(obj))->vtable->klass->name); \
- g_assert_not_reached (); \
+ binary_protocol_missing_remset ((obj), (gpointer)LOAD_VTABLE ((obj)), (char*)(ptr) - (char*)(obj), *(ptr), (gpointer)LOAD_VTABLE(*(ptr)), object_is_pinned (*(ptr))); \
+ if (!object_is_pinned (*(ptr))) \
+ missing_remsets = TRUE; \
} \
} \
} while (0)
/*
- * Check that each object reference inside the area which points into the nursery
- * can be found in the remembered sets.
+ * Check that each object reference which points into the nursery can
+ * be found in the remembered sets.
*/
-static void __attribute__((noinline))
-check_remsets_for_area (char *start, char *end)
+static void
+check_consistency_callback (char *start, size_t size, void *dummy)
{
- GCVTable *vt;
- int type_str = 0, type_rlen = 0, type_bitmap = 0, type_vector = 0, type_lbit = 0, type_complex = 0;
- while (start < end) {
- if (!*(void**)start) {
- start += sizeof (void*); /* should be ALLOC_ALIGN, really */
- continue;
- }
- vt = (GCVTable*)LOAD_VTABLE (start);
- DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
- if (0) {
- MonoObject *obj = (MonoObject*)start;
- g_print ("found at %p (0x%lx): %s.%s\n", start, (long)vt->desc, obj->vtable->klass->name_space, obj->vtable->klass->name);
- }
+ GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
+ DEBUG (8, fprintf (gc_debug_file, "Scanning object %p, vtable: %p (%s)\n", start, vt, vt->klass->name));
-#define SCAN_OBJECT_ACTION COUNT_OBJECT_TYPES
+#define SCAN_OBJECT_ACTION
#include "sgen-scan-object.h"
- }
}
/*
*
* Assumes the world is stopped.
*/
-void
+static void
check_consistency (void)
{
- GCMemSection *section;
-
// Need to add more checks
- // FIXME: Create a general heap enumeration function and use that
+
+ missing_remsets = FALSE;
DEBUG (1, fprintf (gc_debug_file, "Begin heap consistency check...\n"));
// Check that oldspace->newspace pointers are registered with the collector
- for (section = section_list; section; section = section->block.next) {
- if (section->block.role == MEMORY_ROLE_GEN0)
- continue;
- DEBUG (2, fprintf (gc_debug_file, "Scan of old section: %p-%p, size: %d\n", section->data, section->next_data, (int)(section->next_data - section->data)));
- check_remsets_for_area (section->data, section->next_data);
- }
+ major_iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)check_consistency_callback, NULL);
DEBUG (1, fprintf (gc_debug_file, "Heap consistency check done.\n"));
+
+#ifdef BINARY_PROTOCOL
+ if (!binary_protocol_file)
+#endif
+ g_assert (!missing_remsets);
}
/* Check that the reference is valid */
return num_major_gcs;
}
-gint64
+int64_t
mono_gc_get_used_size (void)
{
gint64 tot = 0;
- GCMemSection *section;
LOCK_GC;
tot = los_memory_usage;
- for (section = section_list; section; section = section->block.next) {
- /* this is approximate... */
- tot += section->next_data - section->data;
- }
+ tot += nursery_section->next_data - nursery_section->data;
+ tot += major_get_used_size ();
/* FIXME: account for pinned objects */
UNLOCK_GC;
return tot;
}
-gint64
+int64_t
mono_gc_get_heap_size (void)
{
return total_alloc;
return (MonoObject*) REVEAL_POINTER (*link_addr);
}
+gboolean
+mono_gc_ephemeron_array_add (MonoObject *obj)
+{
+ EphemeronLinkNode *node;
+
+ LOCK_GC;
+
+ node = get_internal_mem (sizeof (EphemeronLinkNode), INTERNAL_MEM_EPHEMERON_LINK);
+ if (!node) {
+ UNLOCK_GC;
+ return FALSE;
+ }
+ node->array = (char*)obj;
+ node->next = ephemeron_list;
+ ephemeron_list = node;
+
+ DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj));
+
+ UNLOCK_GC;
+ return TRUE;
+}
+
void*
mono_gc_make_descr_from_bitmap (gsize *bitmap, int numbits)
{
if (numbits < ((sizeof (*bitmap) * 8) - ROOT_DESC_TYPE_SHIFT)) {
return (void*)MAKE_ROOT_DESC (ROOT_DESC_BITMAP, bitmap [0]);
} else {
- mword complex = alloc_complex_descriptor (bitmap, numbits + 1);
+ mword complex = alloc_complex_descriptor (bitmap, numbits);
return (void*)MAKE_ROOT_DESC (ROOT_DESC_COMPLEX, complex);
}
}
void*
-mono_gc_make_root_descr_user (MonoGCMarkFunc marker)
+mono_gc_make_root_descr_user (MonoGCRootMarkFunc marker)
{
void *descr;
free (addr);
}
+void*
+mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
+{
+ void *result;
+ LOCK_INTERRUPTION;
+ result = func (data);
+ UNLOCK_INTERRUPTION;
+ return result;
+}
+
gboolean
mono_gc_is_gc_thread (void)
{
return result;
}
+#ifdef USER_CONFIG
+
+/* Tries to extract a number from the passed string, taking in to account m, k
+ * and g suffixes */
+static gboolean
+parse_environment_string_extract_number (gchar *str, glong *out)
+{
+ char *endptr;
+ int len = strlen (str), shift = 0;
+ glong val;
+ gboolean is_suffix = FALSE;
+ char suffix;
+
+ switch (str [len - 1]) {
+ case 'g':
+ case 'G':
+ shift += 10;
+ case 'm':
+ case 'M':
+ shift += 10;
+ case 'k':
+ case 'K':
+ shift += 10;
+ is_suffix = TRUE;
+ suffix = str [len - 1];
+ break;
+ }
+
+ errno = 0;
+ val = strtol (str, &endptr, 10);
+
+ if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
+ || (errno != 0 && val == 0) || (endptr == str))
+ return FALSE;
+
+ if (is_suffix) {
+ if (*(endptr + 1)) /* Invalid string. */
+ return FALSE;
+ val <<= shift;
+ }
+
+ *out = val;
+ return TRUE;
+}
+
+#endif
+
void
mono_gc_base_init (void)
{
}
pagesize = mono_pagesize ();
gc_debug_file = stderr;
+
+#ifdef USER_CONFIG
+
+ if ((env = getenv ("MONO_GC_PARAMS"))) {
+ if (g_str_has_prefix (env, "nursery-size")) {
+ int index = 0;
+ long val;
+ while (env [index] && env [index++] != '=')
+ ;
+ if (env [index] && parse_environment_string_extract_number (env
+ + index, &val)) {
+ default_nursery_size = val;
+#ifdef ALIGN_NURSERY
+ if ((val & (val - 1))) {
+ fprintf (stderr, "The nursery size must be a power of two.\n");
+ exit (1);
+ }
+
+ default_nursery_bits = 0;
+ while (1 << (++ default_nursery_bits) != default_nursery_size)
+ ;
+#endif
+ } else {
+ fprintf (stderr, "nursery-size must be an integer.\n");
+ exit (1);
+ }
+ } else {
+ fprintf (stderr, "MONO_GC_PARAMS must be of the form 'nursery-size=N' (where N is an integer, possibly with a k, m or a g suffix).\n");
+ exit (1);
+ }
+ }
+
+#endif
+
+ nursery_size = DEFAULT_NURSERY_SIZE;
+
+ major_init ();
+
if ((env = getenv ("MONO_GC_DEBUG"))) {
opts = g_strsplit (env, ",", -1);
for (ptr = opts; ptr && *ptr; ptr ++) {
nursery_clear_policy = CLEAR_AT_GC;
} else if (!strcmp (opt, "conservative-stack-mark")) {
conservative_stack_mark = TRUE;
+ } else if (!strcmp (opt, "check-scan-starts")) {
+ do_scan_starts_check = TRUE;
} else if (g_str_has_prefix (opt, "heap-dump=")) {
char *filename = strchr (opt, '=') + 1;
nursery_clear_policy = CLEAR_AT_GC;
heap_dump_file = fopen (filename, "w");
if (heap_dump_file)
fprintf (heap_dump_file, "<sgen-dump>\n");
+#ifdef BINARY_PROTOCOL
+ } else if (g_str_has_prefix (opt, "binary-protocol=")) {
+ char *filename = strchr (opt, '=') + 1;
+ binary_protocol_file = fopen (filename, "w");
+#endif
} else {
fprintf (stderr, "Invalid format for the MONO_GC_DEBUG env variable: '%s'\n", env);
fprintf (stderr, "The format is: MONO_GC_DEBUG=[l[:filename]|<option>]+ where l is a debug level 0-9.\n");
enum {
ATYPE_NORMAL,
ATYPE_VECTOR,
+ ATYPE_SMALL,
ATYPE_NUM
};
int tlab_next_addr_var, new_next_var;
int num_params, i;
const char *name = NULL;
+ AllocatorWrapperInfo *info;
#ifdef HAVE_KW_THREAD
int tlab_next_addr_offset = -1;
registered = TRUE;
}
- if (atype == ATYPE_NORMAL) {
+ if (atype == ATYPE_SMALL) {
+ num_params = 1;
+ name = "AllocSmall";
+ } else if (atype == ATYPE_NORMAL) {
num_params = 1;
name = "Alloc";
} else if (atype == ATYPE_VECTOR) {
mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC);
size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
- if (atype == ATYPE_NORMAL) {
+ if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
/* size = vtable->klass->instance_size; */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
mono_mb_emit_stloc (mb, size_var);
/* if (size > MAX_SMALL_OBJ_SIZE) goto slowpath */
- mono_mb_emit_ldloc (mb, size_var);
- mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
- max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
+ if (atype != ATYPE_SMALL) {
+ mono_mb_emit_ldloc (mb, size_var);
+ mono_mb_emit_icon (mb, MAX_SMALL_OBJ_SIZE);
+ max_size_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BGT_S);
+ }
/*
* We need to modify tlab_next, but the JIT only supports reading, so we read
slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
/* Slowpath */
-
- mono_mb_patch_short_branch (mb, max_size_branch);
+ if (atype != ATYPE_SMALL)
+ mono_mb_patch_short_branch (mb, max_size_branch);
mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
/* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
mono_mb_emit_ldarg (mb, 0);
mono_mb_emit_ldloc (mb, size_var);
- if (atype == ATYPE_NORMAL) {
+ if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) {
mono_mb_emit_icall (mb, mono_gc_alloc_obj);
} else if (atype == ATYPE_VECTOR) {
mono_mb_emit_ldarg (mb, 1);
res = mono_mb_create_method (mb, csig, 8);
mono_mb_free (mb);
mono_method_get_header (res)->init_locals = FALSE;
+
+ info = mono_image_alloc0 (mono_defaults.corlib, sizeof (AllocatorWrapperInfo));
+ info->alloc_type = atype;
+ mono_marshal_set_wrapper_info (res, info);
+
return res;
}
#endif
if (collect_before_allocs)
return NULL;
- return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
+ if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE)
+ return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL);
+ else
+ return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL);
#else
return NULL;
#endif
#endif
}
-int
-mono_gc_get_managed_allocator_type (MonoMethod *managed_alloc)
-{
-#ifdef MANAGED_ALLOCATION
- int i;
-
- for (i = 0; i < ATYPE_NUM; ++i)
- if (managed_alloc == alloc_method_cache [i])
- return i;
-#endif
- g_assert_not_reached ();
- return -1;
-}
-
MonoMethod*
mono_gc_get_managed_allocator_by_type (int atype)
{
MonoMethodSignature *sig;
#ifdef MANAGED_WBARRIER
int label_no_wb_1, label_no_wb_2, label_no_wb_3, label_no_wb_4, label_need_wb, label_slow_path;
+#ifndef ALIGN_NURSERY
+ int label_continue_1, label_continue_2, label_no_wb_5;
+ int dereferenced_var;
+#endif
int buffer_var, buffer_index_var, dummy_var;
#ifdef HAVE_KW_THREAD
mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BNE_UN);
#else
- // FIXME:
- g_assert_not_reached ();
-#endif
+ // if (ptr < (nursery_start)) goto continue;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_start);
+ label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
+
+ // if (ptr >= nursery_real_end)) goto continue;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
+ label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
+
+ // Otherwise return
+ label_no_wb_1 = mono_mb_emit_branch (mb, CEE_BR);
+
+ // continue:
+ mono_mb_patch_branch (mb, label_continue_1);
+ mono_mb_patch_branch (mb, label_continue_2);
+
+ // Dereference and store in local var
+ dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_stloc (mb, dereferenced_var);
+
+ // if (*ptr < nursery_start) return;
+ mono_mb_emit_ldloc (mb, dereferenced_var);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_start);
+ label_no_wb_2 = mono_mb_emit_branch (mb, CEE_BLT);
+
+ // if (*ptr >= nursery_end) return;
+ mono_mb_emit_ldloc (mb, dereferenced_var);
+ mono_mb_emit_ptr (mb, (gpointer) nursery_real_end);
+ label_no_wb_5 = mono_mb_emit_branch (mb, CEE_BGE);
+
+#endif
// if (ptr >= stack_end) goto need_wb;
mono_mb_emit_ldarg (mb, 0);
EMIT_TLS_ACCESS (mb, stack_end, stack_end_offset);
mono_mb_patch_branch (mb, label_no_wb_2);
mono_mb_patch_branch (mb, label_no_wb_3);
mono_mb_patch_branch (mb, label_no_wb_4);
+#ifndef ALIGN_NURSERY
+ mono_mb_patch_branch (mb, label_no_wb_5);
+#endif
mono_mb_emit_byte (mb, CEE_RET);
// slow path
projects / mono.git / blobdiff