#include "mono/utils/mono-membar.h"
#ifdef HEAVY_STATISTICS
-static volatile guint64 stat_gc_handles_allocated = 0;
-static volatile guint64 stat_gc_handles_max_allocated = 0;
+static volatile guint32 stat_gc_handles_allocated = 0;
+static volatile guint32 stat_gc_handles_max_allocated = 0;
#endif
#define BUCKETS (32 - MONO_GC_HANDLE_TYPE_SHIFT)
volatile gpointer *volatile entries [BUCKETS];
volatile guint32 capacity;
volatile guint32 slot_hint;
+ volatile guint32 max_index;
guint8 type;
} HandleData;
*offset = index - bucket_size (*bucket) + MIN_BUCKET_SIZE;
}
-static inline gboolean
+static void
+protocol_gchandle_update (int handle_type, gpointer link, gpointer old_value, gpointer new_value)
+{
+ gboolean old = MONO_GC_HANDLE_IS_OBJECT_POINTER (old_value);
+ gboolean new_ = MONO_GC_HANDLE_IS_OBJECT_POINTER (new_value);
+ gboolean track = handle_type == HANDLE_WEAK_TRACK;
+
+ if (!MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type))
+ return;
+
+ if (!old && new_)
+ binary_protocol_dislink_add (link, MONO_GC_REVEAL_POINTER (new_value, TRUE), track);
+ else if (old && !new_)
+ binary_protocol_dislink_remove (link, track);
+ else if (old && new_ && old_value != new_value)
+ binary_protocol_dislink_update (link, MONO_GC_REVEAL_POINTER (new_value, TRUE), track);
+}
+
+/* Returns the new value in the slot, or NULL if the CAS failed. */
+static inline gpointer
try_set_slot (volatile gpointer *slot, GCObject *obj, gpointer old, GCHandleType type)
{
+ gpointer new_;
if (obj)
- return InterlockedCompareExchangePointer (slot, MONO_GC_HANDLE_OBJECT_POINTER (obj, GC_HANDLE_TYPE_IS_WEAK (type)), old) == old;
- return InterlockedCompareExchangePointer (slot, MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (type)), old) == old;
+ new_ = MONO_GC_HANDLE_OBJECT_POINTER (obj, GC_HANDLE_TYPE_IS_WEAK (type));
+ else
+ new_ = MONO_GC_HANDLE_METADATA_POINTER (sgen_client_default_metadata (), GC_HANDLE_TYPE_IS_WEAK (type));
+ SGEN_ASSERT (0, new_, "Why is the occupied bit not set?");
+ if (InterlockedCompareExchangePointer (slot, new_, old) == old) {
+ protocol_gchandle_update (type, (gpointer)slot, old, new_);
+ return new_;
+ }
+ return NULL;
}
/* Try to claim a slot by setting its occupied bit. */
volatile gpointer *link_addr = &(handles->entries [bucket] [offset]);
if (MONO_GC_HANDLE_OCCUPIED (*link_addr))
return FALSE;
- return try_set_slot (link_addr, obj, NULL, handles->type);
+ return try_set_slot (link_addr, obj, NULL, (GCHandleType)handles->type) != NULL;
}
-#define EMPTY_HANDLE_DATA(type) { { NULL }, 0, 0, (type) }
-
-/* weak and weak-track arrays will be allocated in malloc memory
- */
static HandleData gc_handles [] = {
- EMPTY_HANDLE_DATA (HANDLE_WEAK),
- EMPTY_HANDLE_DATA (HANDLE_WEAK_TRACK),
- EMPTY_HANDLE_DATA (HANDLE_NORMAL),
- EMPTY_HANDLE_DATA (HANDLE_PINNED)
+ { { NULL }, 0, 0, 0, (HANDLE_WEAK) },
+ { { NULL }, 0, 0, 0, (HANDLE_WEAK_TRACK) },
+ { { NULL }, 0, 0, 0, (HANDLE_NORMAL) },
+ { { NULL }, 0, 0, 0, (HANDLE_PINNED) }
};
static HandleData *
gc_handles_for_type (GCHandleType type)
{
- g_assert (type < HANDLE_TYPE_MAX);
- return &gc_handles [type];
+ return type < HANDLE_TYPE_MAX ? &gc_handles [type] : NULL;
}
/* This assumes that the world is stopped. */
HandleData *handles = gc_handles_for_type (HANDLE_NORMAL);
size_t bucket, offset;
const guint max_bucket = index_bucket (handles->capacity);
+ guint32 index = 0;
+ const guint32 max_index = handles->max_index;
for (bucket = 0; bucket < max_bucket; ++bucket) {
volatile gpointer *entries = handles->entries [bucket];
- for (offset = 0; offset < bucket_size (bucket); ++offset) {
- volatile gpointer *entry = &entries [offset];
- gpointer hidden = *entry;
- gpointer revealed = MONO_GC_REVEAL_POINTER (hidden, FALSE);
+ for (offset = 0; offset < bucket_size (bucket); ++offset, ++index) {
+ volatile gpointer *entry;
+ gpointer hidden, revealed;
+ /* No need to iterate beyond the largest index ever allocated. */
+ if (index > max_index)
+ return;
+ entry = &entries [offset];
+ hidden = *entry;
+ revealed = MONO_GC_REVEAL_POINTER (hidden, FALSE);
if (!MONO_GC_HANDLE_IS_OBJECT_POINTER (hidden))
continue;
mark_func ((MonoObject **)&revealed, gc_data);
const size_t new_bucket_size = sizeof (**handles->entries) * growth;
if (handles->capacity >= new_capacity)
return;
- entries = g_malloc0 (new_bucket_size);
+ entries = (gpointer *)g_malloc0 (new_bucket_size);
if (handles->type == HANDLE_PINNED)
sgen_register_root ((char *)entries, new_bucket_size, SGEN_DESCRIPTOR_NULL, ROOT_TYPE_PINNED, MONO_ROOT_SOURCE_GC_HANDLE, "pinned gc handles");
+ /* The zeroing of the newly allocated bucket must be complete before storing
+ * the new bucket pointer.
+ */
+ mono_memory_write_barrier ();
if (InterlockedCompareExchangePointer ((volatile gpointer *)&handles->entries [new_bucket], entries, NULL) == NULL) {
+ /* It must not be the case that we succeeded in setting the bucket
+ * pointer, while someone else succeeded in changing the capacity.
+ */
if (InterlockedCompareExchange ((volatile gint32 *)&handles->capacity, new_capacity, old_capacity) != old_capacity)
g_assert_not_reached ();
handles->slot_hint = old_capacity;
- mono_memory_write_barrier ();
return;
}
/* Someone beat us to the allocation. */
guint bucket, offset;
guint32 capacity;
guint32 slot_hint;
+ guint32 max_index;
if (!handles->capacity)
handle_data_grow (handles, 0);
retry:
goto retry;
}
handles->slot_hint = index;
+
+ /*
+ * If a GC happens shortly after a new bucket is allocated, the entire
+ * bucket could be scanned even though it's mostly empty. To avoid this,
+ * we track the maximum index seen so far, so that we can skip the empty
+ * slots.
+ *
+ * Note that we update `max_index` before we even try occupying the
+ * slot. If we did it the other way around and a GC happened in
+ * between, the GC wouldn't know that the slot was occupied. This is
+ * not a huge deal since `obj` is on the stack and thus pinned anyway,
+ * but hopefully some day it won't be anymore.
+ */
+ do {
+ max_index = handles->max_index;
+ if (index <= max_index)
+ break;
+ } while (InterlockedCompareExchange ((volatile gint32 *)&handles->max_index, index, max_index) != max_index);
+
bucketize (index, &bucket, &offset);
if (!try_occupy_slot (handles, bucket, offset, obj, track))
goto retry;
#ifdef HEAVY_STATISTICS
- InterlockedIncrement64 ((volatile gint64 *)&stat_gc_handles_allocated);
+ InterlockedIncrement ((volatile gint32 *)&stat_gc_handles_allocated);
if (stat_gc_handles_allocated > stat_gc_handles_max_allocated)
stat_gc_handles_max_allocated = stat_gc_handles_allocated;
#endif
- if (obj && MONO_GC_HANDLE_TYPE_IS_WEAK (handles->type))
- binary_protocol_dislink_add ((gpointer)&handles->entries [bucket] [offset], obj, track);
/* Ensure that a GC handle cannot be given to another thread without the slot having been set. */
mono_memory_write_barrier ();
res = MONO_GC_HANDLE (index, handles->type);
* This assumes that the world is stopped!
*/
void
-sgen_gchandle_iterate (GCHandleType handle_type, int max_generation, gpointer callback(gpointer, GCHandleType, int, gpointer), gpointer user)
+sgen_gchandle_iterate (GCHandleType handle_type, int max_generation, SgenGCHandleIterateCallback callback, gpointer user)
{
HandleData *handle_data = gc_handles_for_type (handle_type);
size_t bucket, offset;
guint max_bucket = index_bucket (handle_data->capacity);
+ guint32 index = 0;
+ guint32 max_index = handle_data->max_index;
/* If a new bucket has been allocated, but the capacity has not yet been
* increased, nothing can yet have been allocated in the bucket because the
* world is stopped, so we shouldn't miss any handles during iteration.
*/
for (bucket = 0; bucket < max_bucket; ++bucket) {
volatile gpointer *entries = handle_data->entries [bucket];
- for (offset = 0; offset < bucket_size (bucket); ++offset) {
- gpointer hidden = entries [offset];
+ for (offset = 0; offset < bucket_size (bucket); ++offset, ++index) {
+ gpointer hidden;
gpointer result;
/* Table must contain no garbage pointers. */
- gboolean occupied = MONO_GC_HANDLE_OCCUPIED (hidden);
+ gboolean occupied;
+ /* No need to iterate beyond the largest index ever allocated. */
+ if (index > max_index)
+ return;
+ hidden = entries [offset];
+ occupied = MONO_GC_HANDLE_OCCUPIED (hidden);
g_assert (hidden ? occupied : !occupied);
- if (!occupied) // || !MONO_GC_HANDLE_VALID (hidden))
+ if (!occupied)
continue;
result = callback (hidden, handle_type, max_generation, user);
- if (result) {
+ if (result)
SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (result), "Why did the callback return an unoccupied entry?");
- // FIXME: add the dislink_update protocol call here
- } else {
- // FIXME: enable this for weak links
- //binary_protocol_dislink_remove ((gpointer)&handles->entries [bucket] [offset], handles->type == HANDLE_WEAK_TRACK);
- HEAVY_STAT (InterlockedDecrement64 ((volatile gint64 *)&stat_gc_handles_allocated));
- }
+ else
+ HEAVY_STAT (InterlockedDecrement ((volatile gint32 *)&stat_gc_handles_allocated));
+ protocol_gchandle_update (handle_type, (gpointer)&entries [offset], hidden, result);
entries [offset] = result;
}
}
* register, so we need to prevent this from being reordered
* wrt the check.
*/
- mono_gc_dummy_use (obj);
+ sgen_dummy_use (obj);
mono_memory_barrier ();
if (is_weak)
mono_gchandle_get_target (guint32 gchandle)
{
guint index = MONO_GC_HANDLE_SLOT (gchandle);
- guint type = MONO_GC_HANDLE_TYPE (gchandle);
+ GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
HandleData *handles = gc_handles_for_type (type);
+ /* Invalid handles are possible; accessing one should produce NULL. (#34276) */
+ if (!handles)
+ return NULL;
guint bucket, offset;
g_assert (index < handles->capacity);
bucketize (index, &bucket, &offset);
sgen_gchandle_set_target (guint32 gchandle, GCObject *obj)
{
guint index = MONO_GC_HANDLE_SLOT (gchandle);
- guint type = MONO_GC_HANDLE_TYPE (gchandle);
+ GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
HandleData *handles = gc_handles_for_type (type);
- gboolean track = handles->type == HANDLE_WEAK_TRACK;
+ if (!handles)
+ return;
guint bucket, offset;
gpointer slot;
g_assert (index < handles->capacity);
bucketize (index, &bucket, &offset);
-retry:
- slot = handles->entries [bucket] [offset];
- g_assert (MONO_GC_HANDLE_OCCUPIED (slot));
- if (!try_set_slot (&handles->entries [bucket] [offset], obj, slot, MONO_GC_HANDLE_TYPE_IS_WEAK (handles->type)))
- goto retry;
- if (MONO_GC_HANDLE_IS_OBJECT_POINTER (slot))
- binary_protocol_dislink_remove ((gpointer)&handles->entries [bucket] [offset], track);
- if (obj)
- binary_protocol_dislink_add ((gpointer)&handles->entries [bucket] [offset], obj, track);
+ do {
+ slot = handles->entries [bucket] [offset];
+ SGEN_ASSERT (0, MONO_GC_HANDLE_OCCUPIED (slot), "Why are we setting the target on an unoccupied slot?");
+ } while (!try_set_slot (&handles->entries [bucket] [offset], obj, slot, (GCHandleType)handles->type));
}
static gpointer
if (!MONO_GC_HANDLE_OCCUPIED (slot))
return NULL;
if (MONO_GC_HANDLE_IS_OBJECT_POINTER (slot)) {
- GCObject *obj = MONO_GC_REVEAL_POINTER (slot, is_weak);
+ GCObject *obj = (GCObject *)MONO_GC_REVEAL_POINTER (slot, is_weak);
/* See note [dummy use]. */
- mono_gc_dummy_use (obj);
+ sgen_dummy_use (obj);
/*
* FIXME: The compiler could technically not carry a reference to obj around
* at this point and recompute it later, in which case we would still use
}
metadata = MONO_GC_REVEAL_POINTER (slot, is_weak);
/* See note [dummy use]. */
- mono_gc_dummy_use (metadata);
+ sgen_dummy_use (metadata);
if (*slot_addr != slot)
goto retry;
return metadata;
sgen_gchandle_get_metadata (guint32 gchandle)
{
guint index = MONO_GC_HANDLE_SLOT (gchandle);
- guint type = MONO_GC_HANDLE_TYPE (gchandle);
+ GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
HandleData *handles = gc_handles_for_type (type);
+ if (!handles)
+ return NULL;
guint bucket, offset;
if (index >= handles->capacity)
return NULL;
mono_gchandle_free (guint32 gchandle)
{
guint index = MONO_GC_HANDLE_SLOT (gchandle);
- guint type = MONO_GC_HANDLE_TYPE (gchandle);
+ GCHandleType type = MONO_GC_HANDLE_TYPE (gchandle);
HandleData *handles = gc_handles_for_type (type);
+ if (!handles)
+ return;
guint bucket, offset;
+ gpointer slot;
bucketize (index, &bucket, &offset);
- if (index < handles->capacity && MONO_GC_HANDLE_OCCUPIED (handles->entries [bucket] [offset])) {
- if (MONO_GC_HANDLE_TYPE_IS_WEAK (handles->type))
- binary_protocol_dislink_remove ((gpointer)&handles->entries [bucket] [offset], handles->type == HANDLE_WEAK_TRACK);
+ slot = handles->entries [bucket] [offset];
+ if (index < handles->capacity && MONO_GC_HANDLE_OCCUPIED (slot)) {
handles->entries [bucket] [offset] = NULL;
- HEAVY_STAT (InterlockedDecrement64 ((volatile gint64 *)&stat_gc_handles_allocated));
+ protocol_gchandle_update (handles->type, (gpointer)&handles->entries [bucket] [offset], slot, NULL);
+ HEAVY_STAT (InterlockedDecrement ((volatile gint32 *)&stat_gc_handles_allocated));
} else {
/* print a warning? */
}
if (!MONO_GC_HANDLE_VALID (hidden))
return hidden;
- obj = MONO_GC_REVEAL_POINTER (hidden, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type));
+ obj = (GCObject *)MONO_GC_REVEAL_POINTER (hidden, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type));
SGEN_ASSERT (0, obj, "Why is the hidden pointer NULL?");
if (object_older_than (obj, max_generation))
static gpointer
null_link_if (gpointer hidden, GCHandleType handle_type, int max_generation, gpointer user)
{
- /* Strictly speaking, function pointers are not guaranteed to have the same size as data pointers. */
WeakLinkAlivePredicateClosure *closure = (WeakLinkAlivePredicateClosure *)user;
GCObject *obj;
if (!MONO_GC_HANDLE_VALID (hidden))
return hidden;
- obj = MONO_GC_REVEAL_POINTER (hidden, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type));
+ obj = (GCObject *)MONO_GC_REVEAL_POINTER (hidden, MONO_GC_HANDLE_TYPE_IS_WEAK (handle_type));
SGEN_ASSERT (0, obj, "Why is the hidden pointer NULL?");
if (object_older_than (obj, max_generation))
sgen_init_gchandles (void)
{
#ifdef HEAVY_STATISTICS
- mono_counters_register ("GC handles allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_gc_handles_allocated);
- mono_counters_register ("max GC handles allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_gc_handles_max_allocated);
+ mono_counters_register ("GC handles allocated", MONO_COUNTER_GC | MONO_COUNTER_UINT, (void *)&stat_gc_handles_allocated);
+ mono_counters_register ("max GC handles allocated", MONO_COUNTER_GC | MONO_COUNTER_UINT, (void *)&stat_gc_handles_max_allocated);
#endif
}
projects / mono.git / blobdiff