#include <config.h>
-#include <mono/metadata/gc-internal.h>
#include <mono/utils/hazard-pointer.h>
+#include <mono/utils/mono-membar.h>
+#include <mono/utils/mono-memory-model.h>
#include <mono/utils/mono-mmap.h>
+#include <mono/utils/monobitset.h>
+#include <mono/utils/mono-threads.h>
+#include <mono/utils/lock-free-array-queue.h>
+#include <mono/utils/mono-counters.h>
+#include <mono/utils/atomic.h>
#include <mono/io-layer/io-layer.h>
typedef struct {
gpointer p;
MonoHazardousFreeFunc free_func;
+ gboolean might_lock;
} DelayedFreeItem;
/* The hazard table */
#if MONO_SMALL_CONFIG
#define HAZARD_TABLE_MAX_SIZE 256
+#define HAZARD_TABLE_OVERFLOW 4
#else
#define HAZARD_TABLE_MAX_SIZE 16384 /* There cannot be more threads than this number. */
+#define HAZARD_TABLE_OVERFLOW 64
#endif
static volatile int hazard_table_size = 0;
static MonoThreadHazardPointers * volatile hazard_table = NULL;
+/*
+ * Each entry is either 0 or 1, indicating whether that overflow small
+ * ID is busy.
+ */
+static volatile gint32 overflow_busy [HAZARD_TABLE_OVERFLOW];
+
/* The table where we keep pointers to blocks to be freed but that
have to wait because they're guarded by a hazard pointer. */
-static CRITICAL_SECTION delayed_free_table_mutex;
-static GArray *delayed_free_table = NULL;
+static MonoLockFreeArrayQueue delayed_free_queue = MONO_LOCK_FREE_ARRAY_QUEUE_INIT (sizeof (DelayedFreeItem));
/* The table for small ID assignment */
-static CRITICAL_SECTION small_id_mutex;
-static int small_id_table_size = 0;
-static int small_id_next = 0;
+static mono_mutex_t small_id_mutex;
+static int small_id_next;
static int highest_small_id = -1;
-static MonoInternalThread **small_id_table = NULL;
+static MonoBitSet *small_id_table;
+static int hazardous_pointer_count;
/*
* Allocate a small thread id.
* domain_id_alloc() in domain.c and should be merged.
*/
int
-mono_thread_small_id_alloc (MonoInternalThread *thread)
+mono_thread_small_id_alloc (void)
{
- int id = -1, i;
-
- EnterCriticalSection (&small_id_mutex);
-
- if (!small_id_table) {
- small_id_table_size = 2;
- /*
- * Enabling this causes problems, because SGEN doesn't track/update the TLS slot holding
- * the current thread.
- */
- //small_id_table = mono_gc_alloc_fixed (small_id_table_size * sizeof (MonoInternalThread*), mono_gc_make_root_descr_all_refs (small_id_table_size));
- small_id_table = mono_gc_alloc_fixed (small_id_table_size * sizeof (MonoInternalThread*), NULL);
- }
- for (i = small_id_next; i < small_id_table_size; ++i) {
- if (!small_id_table [i]) {
- id = i;
- break;
- }
- }
- if (id == -1) {
- for (i = 0; i < small_id_next; ++i) {
- if (!small_id_table [i]) {
- id = i;
- break;
- }
- }
- }
+ int i, id = -1;
+
+ mono_mutex_lock (&small_id_mutex);
+
+ if (!small_id_table)
+ small_id_table = mono_bitset_new (1, 0);
+
+ id = mono_bitset_find_first_unset (small_id_table, small_id_next - 1);
+ if (id == -1)
+ id = mono_bitset_find_first_unset (small_id_table, -1);
+
if (id == -1) {
- MonoInternalThread **new_table;
- int new_size = small_id_table_size * 2;
- if (new_size >= (1 << 16))
+ MonoBitSet *new_table;
+ if (small_id_table->size * 2 >= (1 << 16))
g_assert_not_reached ();
- id = small_id_table_size;
- //new_table = mono_gc_alloc_fixed (new_size * sizeof (MonoInternalThread*), mono_gc_make_root_descr_all_refs (new_size));
- new_table = mono_gc_alloc_fixed (new_size * sizeof (MonoInternalThread*), NULL);
- memcpy (new_table, small_id_table, small_id_table_size * sizeof (void*));
- mono_gc_free_fixed (small_id_table);
+ new_table = mono_bitset_clone (small_id_table, small_id_table->size * 2);
+ id = mono_bitset_find_first_unset (new_table, small_id_table->size - 1);
+
+ mono_bitset_free (small_id_table);
small_id_table = new_table;
- small_id_table_size = new_size;
}
- thread->small_id = id;
- g_assert (small_id_table [id] == NULL);
- small_id_table [id] = thread;
+
+ g_assert (!mono_bitset_test_fast (small_id_table, id));
+ mono_bitset_set_fast (small_id_table, id);
+
small_id_next++;
- if (small_id_next > small_id_table_size)
+ if (small_id_next >= small_id_table->size)
small_id_next = 0;
g_assert (id < HAZARD_TABLE_MAX_SIZE);
#endif
g_assert (id < hazard_table_size);
- hazard_table [id].hazard_pointers [0] = NULL;
- hazard_table [id].hazard_pointers [1] = NULL;
+ for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
+ hazard_table [id].hazard_pointers [i] = NULL;
}
if (id > highest_small_id) {
mono_memory_write_barrier ();
}
- LeaveCriticalSection (&small_id_mutex);
+ mono_mutex_unlock (&small_id_mutex);
return id;
}
void
mono_thread_small_id_free (int id)
{
- g_assert (id >= 0 && id < small_id_table_size);
- g_assert (small_id_table [id] != NULL);
+ /* MonoBitSet operations are not atomic. */
+ mono_mutex_lock (&small_id_mutex);
- small_id_table [id] = NULL;
+ g_assert (id >= 0 && id < small_id_table->size);
+ g_assert (mono_bitset_test_fast (small_id_table, id));
+ mono_bitset_clear_fast (small_id_table, id);
+
+ mono_mutex_unlock (&small_id_mutex);
}
static gboolean
is_pointer_hazardous (gpointer p)
{
- int i;
+ int i, j;
int highest = highest_small_id;
g_assert (highest < hazard_table_size);
for (i = 0; i <= highest; ++i) {
- if (hazard_table [i].hazard_pointers [0] == p
- || hazard_table [i].hazard_pointers [1] == p)
- return TRUE;
+ for (j = 0; j < HAZARD_POINTER_COUNT; ++j) {
+ if (hazard_table [i].hazard_pointers [j] == p)
+ return TRUE;
+ LOAD_LOAD_FENCE;
+ }
}
return FALSE;
MonoThreadHazardPointers*
mono_hazard_pointer_get (void)
{
- MonoInternalThread *current_thread = mono_thread_internal_current ();
+ int small_id = mono_thread_info_get_small_id ();
- if (!(current_thread && current_thread->small_id >= 0)) {
+ if (small_id < 0) {
static MonoThreadHazardPointers emerg_hazard_table;
- g_warning ("Thread %p may have been prematurely finalized", current_thread);
+ g_warning ("Thread %p may have been prematurely finalized", (gpointer)mono_native_thread_id_get ());
return &emerg_hazard_table;
}
- return &hazard_table [current_thread->small_id];
+ return &hazard_table [small_id];
}
/* Can be called with hp==NULL, in which case it acts as an ordinary
return p;
}
-static void
-try_free_delayed_free_item (int index)
+int
+mono_hazard_pointer_save_for_signal_handler (void)
{
- if (delayed_free_table->len > index) {
- DelayedFreeItem item = { NULL, NULL };
-
- EnterCriticalSection (&delayed_free_table_mutex);
- /* We have to check the length again because another
- thread might have freed an item before we acquired
- the lock. */
- if (delayed_free_table->len > index) {
- item = g_array_index (delayed_free_table, DelayedFreeItem, index);
-
- if (!is_pointer_hazardous (item.p))
- g_array_remove_index_fast (delayed_free_table, index);
- else
- item.p = NULL;
- }
- LeaveCriticalSection (&delayed_free_table_mutex);
+ int small_id, i;
+ MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
+ MonoThreadHazardPointers *hp_overflow;
+
+ for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
+ if (hp->hazard_pointers [i])
+ goto search;
+ return -1;
+
+ search:
+ for (small_id = 0; small_id < HAZARD_TABLE_OVERFLOW; ++small_id) {
+ if (!overflow_busy [small_id])
+ break;
+ }
+
+ /*
+ * If this assert fails we don't have enough overflow slots.
+ * We should contemplate adding them dynamically. If we can
+ * make mono_thread_small_id_alloc() lock-free we can just
+ * allocate them on-demand.
+ */
+ g_assert (small_id < HAZARD_TABLE_OVERFLOW);
+
+ if (InterlockedCompareExchange (&overflow_busy [small_id], 1, 0) != 0)
+ goto search;
+
+ hp_overflow = &hazard_table [small_id];
+
+ for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
+ g_assert (!hp_overflow->hazard_pointers [i]);
+ *hp_overflow = *hp;
+
+ mono_memory_write_barrier ();
+
+ memset (hp, 0, sizeof (MonoThreadHazardPointers));
+
+ return small_id;
+}
+
+void
+mono_hazard_pointer_restore_for_signal_handler (int small_id)
+{
+ MonoThreadHazardPointers *hp = mono_hazard_pointer_get ();
+ MonoThreadHazardPointers *hp_overflow;
+ int i;
+
+ if (small_id < 0)
+ return;
+
+ g_assert (small_id < HAZARD_TABLE_OVERFLOW);
+ g_assert (overflow_busy [small_id]);
+
+ for (i = 0; i < HAZARD_POINTER_COUNT; ++i)
+ g_assert (!hp->hazard_pointers [i]);
+
+ hp_overflow = &hazard_table [small_id];
+
+ *hp = *hp_overflow;
+
+ mono_memory_write_barrier ();
+
+ memset (hp_overflow, 0, sizeof (MonoThreadHazardPointers));
+
+ mono_memory_write_barrier ();
+
+ overflow_busy [small_id] = 0;
+}
+
+static gboolean
+try_free_delayed_free_item (gboolean lock_free_context)
+{
+ DelayedFreeItem item;
+ gboolean popped = mono_lock_free_array_queue_pop (&delayed_free_queue, &item);
- if (item.p != NULL)
- item.free_func (item.p);
+ if (!popped)
+ return FALSE;
+
+ if ((lock_free_context && item.might_lock) || (is_pointer_hazardous (item.p))) {
+ mono_lock_free_array_queue_push (&delayed_free_queue, &item);
+ return FALSE;
}
+
+ item.free_func (item.p);
+
+ return TRUE;
}
void
-mono_thread_hazardous_free_or_queue (gpointer p, MonoHazardousFreeFunc free_func)
+mono_thread_hazardous_free_or_queue (gpointer p, MonoHazardousFreeFunc free_func,
+ gboolean free_func_might_lock, gboolean lock_free_context)
{
int i;
+ if (lock_free_context)
+ g_assert (!free_func_might_lock);
+ if (free_func_might_lock)
+ g_assert (!lock_free_context);
+
/* First try to free a few entries in the delayed free
table. */
- for (i = 2; i >= 0; --i)
- try_free_delayed_free_item (i);
+ for (i = 0; i < 3; ++i)
+ try_free_delayed_free_item (lock_free_context);
/* Now see if the pointer we're freeing is hazardous. If it
isn't, free it. Otherwise put it in the delay list. */
if (is_pointer_hazardous (p)) {
- DelayedFreeItem item = { p, free_func };
+ DelayedFreeItem item = { p, free_func, free_func_might_lock };
- ++mono_stats.hazardous_pointer_count;
+ ++hazardous_pointer_count;
- EnterCriticalSection (&delayed_free_table_mutex);
- g_array_append_val (delayed_free_table, item);
- LeaveCriticalSection (&delayed_free_table_mutex);
- } else
+ mono_lock_free_array_queue_push (&delayed_free_queue, &item);
+ } else {
free_func (p);
+ }
}
void
mono_thread_hazardous_try_free_all (void)
{
- int len;
- int i;
-
- if (!delayed_free_table)
- return;
-
- len = delayed_free_table->len;
+ while (try_free_delayed_free_item (FALSE))
+ ;
+}
- for (i = len - 1; i >= 0; --i)
- try_free_delayed_free_item (i);
+void
+mono_thread_hazardous_try_free_some (void)
+{
+ int i;
+ for (i = 0; i < 10; ++i)
+ try_free_delayed_free_item (FALSE);
}
void
mono_thread_smr_init (void)
{
- MONO_GC_REGISTER_ROOT_FIXED (small_id_table);
+ int i;
- InitializeCriticalSection(&delayed_free_table_mutex);
- InitializeCriticalSection(&small_id_mutex);
+ mono_mutex_init_recursive(&small_id_mutex);
+ mono_counters_register ("Hazardous pointers", MONO_COUNTER_JIT | MONO_COUNTER_INT, &hazardous_pointer_count);
- delayed_free_table = g_array_new (FALSE, FALSE, sizeof (DelayedFreeItem));
+ for (i = 0; i < HAZARD_TABLE_OVERFLOW; ++i) {
+ int small_id = mono_thread_small_id_alloc ();
+ g_assert (small_id == i);
+ }
}
void
{
mono_thread_hazardous_try_free_all ();
- g_array_free (delayed_free_table, TRUE);
- delayed_free_table = NULL;
-
+ mono_lock_free_array_queue_cleanup (&delayed_free_queue);
+
+ /*FIXME, can't we release the small id table here?*/
}
projects / mono.git / blobdiff