-/*
- * sgen-los.c: Large objects space.
+/**
+ * \file
+ * Large objects space.
*
* Author:
* Paolo Molaro (lupus@ximian.com)
* Copyright 2003-2010 Novell, Inc.
* Copyright (C) 2012 Xamarin Inc
*
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License 2.0 as published by the Free Software Foundation;
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License 2.0 along with this library; if not, write to the Free
- * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ * Licensed under the MIT license. See LICENSE file in the project root for full license information.
*/
#include "config.h"
unsigned char *free_chunk_map;
};
+/* We allow read only access on the list while sweep is not running */
LOSObject *los_object_list = NULL;
+/* Memory used by LOS objects */
mword los_memory_usage = 0;
+/* Total memory used by the LOS allocator */
+mword los_memory_usage_total = 0;
static LOSSection *los_sections = NULL;
static LOSFreeChunks *los_fast_free_lists [LOS_NUM_FAST_SIZES]; /* 0 is for larger sizes */
return free_chunks;
}
+static LOSObject*
+randomize_los_object_start (gpointer addr, size_t obj_size, size_t alloced_size, size_t addr_alignment)
+{
+ size_t offset = 0;
+ if (alloced_size != obj_size) {
+ /*
+ * We want to get a random offset between 0 and (alloced_size - obj_size)
+ * We do a prime multiplication to avoid usage of functions which might not
+ * be thread/signal safe (like rand ()). We subtract 1 to avoid common
+ * power by 2 factors.
+ */
+ offset = SGEN_ALIGN_DOWN ((((size_t)addr - 1) * 2654435761u) % (alloced_size - obj_size));
+ }
+ SGEN_ASSERT (0, (alloced_size - obj_size) < addr_alignment, "Why are we wasting one entire chunk for a los object ?");
+ /* Randomize the location within the reserved chunks to improve cache performance */
+ return (LOSObject*)((guint8*)addr + offset);
+
+}
+
static LOSObject*
get_los_section_memory (size_t size)
{
LOSSection *section;
LOSFreeChunks *free_chunks;
size_t num_chunks;
+ size_t obj_size = size;
- size += LOS_CHUNK_SIZE - 1;
- size &= ~(LOS_CHUNK_SIZE - 1);
+ size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
num_chunks = size >> LOS_CHUNK_BITS;
free_chunks = get_from_size_list (&los_fast_free_lists [0], size);
}
- if (free_chunks)
- return (LOSObject*)free_chunks;
+ if (free_chunks) {
+ return randomize_los_object_start (free_chunks, obj_size, size, LOS_CHUNK_SIZE);
+ }
if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
return NULL;
- section = sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
+ section = (LOSSection *)sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
if (!section)
return NULL;
section->next = los_sections;
los_sections = section;
+ los_memory_usage_total += LOS_SECTION_SIZE;
++los_num_sections;
goto retry;
LOSSection *section = LOS_SECTION_FOR_OBJ (obj);
size_t num_chunks, i, start_index;
- size += LOS_CHUNK_SIZE - 1;
- size &= ~(LOS_CHUNK_SIZE - 1);
+ size = SGEN_ALIGN_UP_TO (size, LOS_CHUNK_SIZE);
num_chunks = size >> LOS_CHUNK_BITS;
section->free_chunk_map [i] = 1;
}
- add_free_chunk ((LOSFreeChunks*)obj, size);
+ add_free_chunk ((LOSFreeChunks*)SGEN_ALIGN_DOWN_TO ((mword)obj, LOS_CHUNK_SIZE), size);
}
-static int pagesize;
-
void
sgen_los_free_object (LOSObject *obj)
{
- SGEN_ASSERT (0, !obj->cardtable_mod_union, "We should never free a LOS object with a mod-union table.");
+ if (obj->cardtable_mod_union)
+ sgen_card_table_free_mod_union (obj->cardtable_mod_union, (char*)obj->data, sgen_los_object_size (obj));
#ifndef LOS_DUMMY
mword size = sgen_los_object_size (obj);
los_num_objects--;
#ifdef USE_MALLOC
- free (obj);
+ g_free (obj);
#else
if (size > LOS_SECTION_OBJECT_LIMIT) {
- if (!pagesize)
- pagesize = mono_pagesize ();
+ int pagesize = mono_pagesize ();
size += sizeof (LOSObject);
- size += pagesize - 1;
- size &= ~(pagesize - 1);
- sgen_free_os_memory (obj, size, SGEN_ALLOC_HEAP);
+ size = SGEN_ALIGN_UP_TO (size, pagesize);
+ sgen_free_os_memory ((gpointer)SGEN_ALIGN_DOWN_TO ((mword)obj, pagesize), size, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
+ los_memory_usage_total -= size;
sgen_memgov_release_space (size, SPACE_LOS);
} else {
free_los_section_memory (obj, size + sizeof (LOSObject));
los_segment_index += size + sizeof (LOSObject);
g_assert (los_segment_index <= LOS_SEGMENT_SIZE);
#else
- sgen_ensure_free_space (size);
+ sgen_ensure_free_space (size, GENERATION_OLD);
#ifdef USE_MALLOC
- obj = malloc (size + sizeof (LOSObject));
+ obj = g_malloc (size + sizeof (LOSObject));
memset (obj, 0, size + sizeof (LOSObject));
#else
if (size > LOS_SECTION_OBJECT_LIMIT) {
- size_t alloc_size = size;
- if (!pagesize)
- pagesize = mono_pagesize ();
- alloc_size += sizeof (LOSObject);
- alloc_size += pagesize - 1;
- alloc_size &= ~(pagesize - 1);
+ size_t obj_size = size + sizeof (LOSObject);
+ int pagesize = mono_pagesize ();
+ size_t alloc_size = SGEN_ALIGN_UP_TO (obj_size, pagesize);
if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
- obj = sgen_alloc_os_memory (alloc_size, SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE, NULL);
+ obj = (LOSObject *)sgen_alloc_os_memory (alloc_size, (SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE), NULL, MONO_MEM_ACCOUNT_SGEN_LOS);
+ if (obj) {
+ los_memory_usage_total += alloc_size;
+ obj = randomize_los_object_start (obj, obj_size, alloc_size, pagesize);
+ }
}
} else {
obj = get_los_section_memory (size + sizeof (LOSObject));
*vtslot = vtable;
sgen_update_heap_boundaries ((mword)obj->data, (mword)obj->data + size);
obj->next = los_object_list;
+ /*
+ * We need a memory barrier so we don't expose as head of the los object list
+ * a LOSObject that doesn't have its fields initialized.
+ */
+ mono_memory_write_barrier ();
los_object_list = obj;
los_memory_usage += size;
los_num_objects++;
for (bigobj = los_object_list; bigobj;) {
SGEN_ASSERT (0, !SGEN_OBJECT_IS_PINNED (bigobj->data), "Who pinned a LOS object?");
- if (bigobj->cardtable_mod_union) {
- sgen_card_table_free_mod_union (bigobj->cardtable_mod_union, (char*)bigobj->data, sgen_los_object_size (bigobj));
- bigobj->cardtable_mod_union = NULL;
- }
-
if (sgen_los_object_is_pinned (bigobj->data)) {
+ if (bigobj->cardtable_mod_union) {
+ mword obj_size = sgen_los_object_size (bigobj);
+ mword num_cards = sgen_card_table_number_of_cards_in_range ((mword) bigobj->data, obj_size);
+ memset (bigobj->cardtable_mod_union, 0, num_cards);
+ }
+
sgen_los_unpin_object (bigobj->data);
sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
} else {
prev->next = next;
else
los_sections = next;
- sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP);
+ sgen_free_os_memory (section, LOS_SECTION_SIZE, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_LOS);
sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
section = next;
--los_num_sections;
+ los_memory_usage_total -= LOS_SECTION_SIZE;
continue;
}
{
LOSObject *obj;
- *start = NULL;
+ if (start)
+ *start = NULL;
for (obj = los_object_list; obj; obj = obj->next) {
char *end = (char*)obj->data + sgen_los_object_size (obj);
if (ptr >= (char*)obj->data && ptr < end) {
- *start = (char*)obj->data;
+ if (start)
+ *start = (char*)obj->data;
return TRUE;
}
}
if (mod_union)
return mod_union;
mod_union = sgen_card_table_alloc_mod_union ((char*)obj->data, size);
- other = SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
+ other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&obj->cardtable_mod_union, mod_union, NULL);
if (!other) {
SGEN_ASSERT (0, obj->cardtable_mod_union == mod_union, "Why did CAS not replace?");
return mod_union;
}
void
-sgen_los_scan_card_table (gboolean mod_union, ScanCopyContext ctx)
+sgen_los_scan_card_table (CardTableScanType scan_type, ScanCopyContext ctx, int job_index, int job_split_count)
{
LOSObject *obj;
+ int i = 0;
- for (obj = los_object_list; obj; obj = obj->next) {
+ binary_protocol_los_card_table_scan_start (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
+ for (obj = los_object_list; obj; obj = obj->next, i++) {
+ mword num_cards = 0;
guint8 *cards;
+ if (i % job_split_count != job_index)
+ continue;
+
if (!SGEN_OBJECT_HAS_REFERENCES (obj->data))
continue;
- if (mod_union) {
+ if (scan_type & CARDTABLE_SCAN_MOD_UNION) {
if (!sgen_los_object_is_pinned (obj->data))
continue;
+ if (!obj->cardtable_mod_union)
+ continue;
+
cards = get_cardtable_mod_union_for_object (obj);
g_assert (cards);
+ if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN) {
+ guint8 *cards_preclean;
+ mword obj_size = sgen_los_object_size (obj);
+ num_cards = sgen_card_table_number_of_cards_in_range ((mword) obj->data, obj_size);
+ cards_preclean = (guint8 *)sgen_alloc_internal_dynamic (num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION, TRUE);
+
+ sgen_card_table_preclean_mod_union (cards, cards_preclean, num_cards);
+
+ cards = cards_preclean;
+ }
} else {
cards = NULL;
}
- sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, mod_union, ctx);
+ sgen_cardtable_scan_object (obj->data, sgen_los_object_size (obj), cards, ctx);
+
+ if (scan_type == CARDTABLE_SCAN_MOD_UNION_PRECLEAN)
+ sgen_free_internal_dynamic (cards, num_cards, INTERNAL_MEM_CARDTABLE_MOD_UNION);
}
+ binary_protocol_los_card_table_scan_end (sgen_timestamp (), scan_type & CARDTABLE_SCAN_MOD_UNION);
}
void
binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
}
+gboolean
+sgen_los_pin_object_par (GCObject *data)
+{
+ LOSObject *obj = sgen_los_header_for_object (data);
+ mword old_size = obj->size;
+ if (old_size & 1)
+ return FALSE;
+#if SIZEOF_VOID_P == 4
+ old_size = InterlockedCompareExchange ((volatile gint32*)&obj->size, old_size | 1, old_size);
+#else
+ old_size = InterlockedCompareExchange64 ((volatile gint64*)&obj->size, old_size | 1, old_size);
+#endif
+ if (old_size & 1)
+ return FALSE;
+ binary_protocol_pin (data, (gpointer)SGEN_LOAD_VTABLE (data), sgen_safe_object_get_size (data));
+ return TRUE;
+}
+
static void
sgen_los_unpin_object (GCObject *data)
{
{
LOSObject *obj = sgen_los_header_for_object (mono_obj);
guint8 *mod_union = get_cardtable_mod_union_for_object (obj);
- size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer ((char*)obj));
+ /* The LOSObject structure is not represented within the card space */
+ size_t offset = sgen_card_table_get_card_offset ((char*)ptr, (char*)sgen_card_table_align_pointer((char*)mono_obj));
SGEN_ASSERT (0, mod_union, "FIXME: optionally allocate the mod union if it's not here and CAS it in.");
- SGEN_ASSERT (0, (char*)obj == (char*)sgen_card_table_align_pointer ((char*)obj), "Why are LOS objects not card aligned?");
mod_union [offset] = 1;
}
projects / mono.git / blobdiff