Merge pull request #2763 from esdrubal/mono-symbolicate-standalone2
[mono.git] / mono / metadata / threadpool-ms.c
index b9d9f941c0cb13b2d380bd5418c286df5b3dcdc3..32d93e6e522490a4564332d022e0ddb013ca251f 100644 (file)
@@ -27,7 +27,7 @@
 
 #include <mono/metadata/class-internals.h>
 #include <mono/metadata/exception.h>
-#include <mono/metadata/gc-internal.h>
+#include <mono/metadata/gc-internals.h>
 #include <mono/metadata/object.h>
 #include <mono/metadata/object-internals.h>
 #include <mono/metadata/threadpool-ms.h>
@@ -37,7 +37,7 @@
 #include <mono/utils/mono-complex.h>
 #include <mono/utils/mono-lazy-init.h>
 #include <mono/utils/mono-logger.h>
-#include <mono/utils/mono-logger-internal.h>
+#include <mono/utils/mono-logger-internals.h>
 #include <mono/utils/mono-proclib.h>
 #include <mono/utils/mono-threads.h>
 #include <mono/utils/mono-time.h>
 #define CPU_USAGE_LOW 80
 #define CPU_USAGE_HIGH 95
 
-#define MONITOR_INTERVAL 100 // ms
+#define MONITOR_INTERVAL 500 // ms
 #define MONITOR_MINIMAL_LIFETIME 60 * 1000 // ms
 
+#define WORKER_CREATION_MAX_PER_SEC 10
+
 /* The exponent to apply to the gain. 1.0 means to use linear gain,
  * higher values will enhance large moves and damp small ones.
  * default: 2.0 */
@@ -125,12 +127,16 @@ typedef struct {
        ThreadPoolCounter counters;
 
        GPtrArray *domains; // ThreadPoolDomain* []
-       mono_mutex_t domains_lock;
+       MonoCoopMutex domains_lock;
 
        GPtrArray *working_threads; // ThreadPoolWorkingThread* []
        gint32 parked_threads_count;
-       mono_cond_t parked_threads_cond;
-       mono_mutex_t active_threads_lock; /* protect access to working_threads and parked_threads */
+       MonoCoopCond parked_threads_cond;
+       MonoCoopMutex active_threads_lock; /* protect access to working_threads and parked_threads */
+
+       guint32 worker_creation_current_second;
+       guint32 worker_creation_current_count;
+       MonoCoopMutex worker_creation_lock;
 
        gint32 heuristic_completions;
        guint32 heuristic_sample_start;
@@ -138,7 +144,7 @@ typedef struct {
        guint32 heuristic_last_adjustment; // ms
        guint32 heuristic_adjustment_interval; // ms
        ThreadPoolHillClimbing heuristic_hill_climbing;
-       mono_mutex_t heuristic_lock;
+       MonoCoopMutex heuristic_lock;
 
        gint32 limit_worker_min;
        gint32 limit_worker_max;
@@ -221,11 +227,11 @@ rand_create (void)
 static guint32
 rand_next (gpointer *handle, guint32 min, guint32 max)
 {
+       MonoError error;
        guint32 val;
-       if (!mono_rand_try_get_uint32 (handle, &val, min, max)) {
-               // FIXME handle error
-               g_assert_not_reached ();
-       }
+       mono_rand_try_get_uint32 (handle, &val, min, max, &error);
+       // FIXME handle error
+       mono_error_assert_ok (&error);
        return val;
 }
 
@@ -248,15 +254,18 @@ initialize (void)
        g_assert (threadpool);
 
        threadpool->domains = g_ptr_array_new ();
-       mono_mutex_init_recursive (&threadpool->domains_lock);
+       mono_coop_mutex_init (&threadpool->domains_lock);
 
        threadpool->parked_threads_count = 0;
-       mono_cond_init (&threadpool->parked_threads_cond, NULL);
+       mono_coop_cond_init (&threadpool->parked_threads_cond);
        threadpool->working_threads = g_ptr_array_new ();
-       mono_mutex_init_recursive (&threadpool->active_threads_lock);
+       mono_coop_mutex_init (&threadpool->active_threads_lock);
+
+       threadpool->worker_creation_current_second = -1;
+       mono_coop_mutex_init (&threadpool->worker_creation_lock);
 
        threadpool->heuristic_adjustment_interval = 10;
-       mono_mutex_init (&threadpool->heuristic_lock);
+       mono_coop_mutex_init (&threadpool->heuristic_lock);
 
        mono_rand_open ();
 
@@ -295,7 +304,12 @@ initialize (void)
        threads_count = mono_cpu_count () * threads_per_cpu;
 
        threadpool->limit_worker_min = threadpool->limit_io_min = threads_count;
+
+#if defined (PLATFORM_ANDROID) || defined (HOST_IOS)
+       threadpool->limit_worker_max = threadpool->limit_io_max = CLAMP (threads_count * 100, MIN (threads_count, 200), MAX (threads_count, 200));
+#else
        threadpool->limit_worker_max = threadpool->limit_io_max = threads_count * 100;
+#endif
 
        threadpool->counters._.max_working = threadpool->limit_worker_min;
 
@@ -315,27 +329,23 @@ cleanup (void)
         * cleaning up only if the runtime is shutting down */
        g_assert (mono_runtime_is_shutting_down ());
 
-       MONO_PREPARE_BLOCKING;
        while (monitor_status != MONITOR_STATUS_NOT_RUNNING)
-               g_usleep (1000);
-       MONO_FINISH_BLOCKING;
+               mono_thread_info_sleep (1, NULL);
 
-       MONO_PREPARE_BLOCKING;
-       mono_mutex_lock (&threadpool->active_threads_lock);
-       MONO_FINISH_BLOCKING;
+       mono_coop_mutex_lock (&threadpool->active_threads_lock);
 
        /* stop all threadpool->working_threads */
        for (i = 0; i < threadpool->working_threads->len; ++i)
                worker_kill ((ThreadPoolWorkingThread*) g_ptr_array_index (threadpool->working_threads, i));
 
        /* unpark all threadpool->parked_threads */
-       mono_cond_broadcast (&threadpool->parked_threads_cond);
+       mono_coop_cond_broadcast (&threadpool->parked_threads_cond);
 
-       mono_mutex_unlock (&threadpool->active_threads_lock);
+       mono_coop_mutex_unlock (&threadpool->active_threads_lock);
 }
 
-void
-mono_threadpool_ms_enqueue_work_item (MonoDomain *domain, MonoObject *work_item)
+gboolean
+mono_threadpool_ms_enqueue_work_item (MonoDomain *domain, MonoObject *work_item, MonoError *error)
 {
        static MonoClass *threadpool_class = NULL;
        static MonoMethod *unsafe_queue_custom_work_item_method = NULL;
@@ -343,11 +353,11 @@ mono_threadpool_ms_enqueue_work_item (MonoDomain *domain, MonoObject *work_item)
        MonoBoolean f;
        gpointer args [2];
 
+       mono_error_init (error);
        g_assert (work_item);
 
        if (!threadpool_class)
-               threadpool_class = mono_class_from_name (mono_defaults.corlib, "System.Threading", "ThreadPool");
-       g_assert (threadpool_class);
+               threadpool_class = mono_class_load_from_name (mono_defaults.corlib, "System.Threading", "ThreadPool");
 
        if (!unsafe_queue_custom_work_item_method)
                unsafe_queue_custom_work_item_method = mono_class_get_method_from_name (threadpool_class, "UnsafeQueueCustomWorkItem", 2);
@@ -360,17 +370,24 @@ mono_threadpool_ms_enqueue_work_item (MonoDomain *domain, MonoObject *work_item)
 
        current_domain = mono_domain_get ();
        if (current_domain == domain) {
-               mono_runtime_invoke (unsafe_queue_custom_work_item_method, NULL, args, NULL);
+               mono_runtime_invoke_checked (unsafe_queue_custom_work_item_method, NULL, args, error);
+               return_val_if_nok (error, FALSE);
        } else {
                mono_thread_push_appdomain_ref (domain);
                if (mono_domain_set (domain, FALSE)) {
-                       mono_runtime_invoke (unsafe_queue_custom_work_item_method, NULL, args, NULL);
+                       mono_runtime_invoke_checked (unsafe_queue_custom_work_item_method, NULL, args, error);
+                       if (!is_ok (error)) {
+                               mono_thread_pop_appdomain_ref ();
+                               return FALSE;
+                       }
                        mono_domain_set (current_domain, TRUE);
                }
                mono_thread_pop_appdomain_ref ();
        }
+       return TRUE;
 }
 
+/* LOCKING: threadpool->domains_lock must be held */
 static void
 domain_add (ThreadPoolDomain *tpdomain)
 {
@@ -378,31 +395,25 @@ domain_add (ThreadPoolDomain *tpdomain)
 
        g_assert (tpdomain);
 
-       mono_mutex_lock (&threadpool->domains_lock);
        len = threadpool->domains->len;
        for (i = 0; i < len; ++i) {
                if (g_ptr_array_index (threadpool->domains, i) == tpdomain)
                        break;
        }
+
        if (i == len)
                g_ptr_array_add (threadpool->domains, tpdomain);
-       mono_mutex_unlock (&threadpool->domains_lock);
 }
 
+/* LOCKING: threadpool->domains_lock must be held */
 static gboolean
 domain_remove (ThreadPoolDomain *tpdomain)
 {
-       gboolean res;
-
        g_assert (tpdomain);
-
-       mono_mutex_lock (&threadpool->domains_lock);
-       res = g_ptr_array_remove (threadpool->domains, tpdomain);
-       mono_mutex_unlock (&threadpool->domains_lock);
-
-       return res;
+       return g_ptr_array_remove (threadpool->domains, tpdomain);
 }
 
+/* LOCKING: threadpool->domains_lock must be held */
 static ThreadPoolDomain *
 domain_get (MonoDomain *domain, gboolean create)
 {
@@ -411,20 +422,18 @@ domain_get (MonoDomain *domain, gboolean create)
 
        g_assert (domain);
 
-       mono_mutex_lock (&threadpool->domains_lock);
        for (i = 0; i < threadpool->domains->len; ++i) {
-               ThreadPoolDomain *tmp = g_ptr_array_index (threadpool->domains, i);
-               if (tmp->domain == domain) {
-                       tpdomain = tmp;
-                       break;
-               }
+               tpdomain = (ThreadPoolDomain *)g_ptr_array_index (threadpool->domains, i);
+               if (tpdomain->domain == domain)
+                       return tpdomain;
        }
-       if (!tpdomain && create) {
+
+       if (create) {
                tpdomain = g_new0 (ThreadPoolDomain, 1);
                tpdomain->domain = domain;
                domain_add (tpdomain);
        }
-       mono_mutex_unlock (&threadpool->domains_lock);
+
        return tpdomain;
 }
 
@@ -434,31 +443,28 @@ domain_free (ThreadPoolDomain *tpdomain)
        g_free (tpdomain);
 }
 
+/* LOCKING: threadpool->domains_lock must be held */
 static gboolean
 domain_any_has_request (void)
 {
-       gboolean res = FALSE;
        guint i;
 
-       mono_mutex_lock (&threadpool->domains_lock);
        for (i = 0; i < threadpool->domains->len; ++i) {
-               ThreadPoolDomain *tmp = g_ptr_array_index (threadpool->domains, i);
-               if (tmp->outstanding_request > 0) {
-                       res = TRUE;
-                       break;
-               }
+               ThreadPoolDomain *tmp = (ThreadPoolDomain *)g_ptr_array_index (threadpool->domains, i);
+               if (tmp->outstanding_request > 0)
+                       return TRUE;
        }
-       mono_mutex_unlock (&threadpool->domains_lock);
-       return res;
+
+       return FALSE;
 }
 
+/* LOCKING: threadpool->domains_lock must be held */
 static ThreadPoolDomain *
 domain_get_next (ThreadPoolDomain *current)
 {
        ThreadPoolDomain *tpdomain = NULL;
        guint len;
 
-       mono_mutex_lock (&threadpool->domains_lock);
        len = threadpool->domains->len;
        if (len > 0) {
                guint i, current_idx = -1;
@@ -472,23 +478,23 @@ domain_get_next (ThreadPoolDomain *current)
                        g_assert (current_idx >= 0);
                }
                for (i = current_idx + 1; i < len + current_idx + 1; ++i) {
-                       ThreadPoolDomain *tmp = g_ptr_array_index (threadpool->domains, i % len);
+                       ThreadPoolDomain *tmp = (ThreadPoolDomain *)g_ptr_array_index (threadpool->domains, i % len);
                        if (tmp->outstanding_request > 0) {
                                tpdomain = tmp;
                                break;
                        }
                }
        }
-       mono_mutex_unlock (&threadpool->domains_lock);
+
        return tpdomain;
 }
 
 static void
 worker_wait_interrupt (gpointer data)
 {
-       mono_mutex_lock (&threadpool->active_threads_lock);
-       mono_cond_signal (&threadpool->parked_threads_cond);
-       mono_mutex_unlock (&threadpool->active_threads_lock);
+       mono_coop_mutex_lock (&threadpool->active_threads_lock);
+       mono_coop_cond_signal (&threadpool->parked_threads_cond);
+       mono_coop_mutex_unlock (&threadpool->active_threads_lock);
 }
 
 /* return TRUE if timeout, FALSE otherwise (worker unpark or interrupt) */
@@ -501,9 +507,7 @@ worker_park (void)
 
        mono_gc_set_skip_thread (TRUE);
 
-       MONO_PREPARE_BLOCKING;
-
-       mono_mutex_lock (&threadpool->active_threads_lock);
+       mono_coop_mutex_lock (&threadpool->active_threads_lock);
 
        if (!mono_runtime_is_shutting_down ()) {
                static gpointer rand_handle = NULL;
@@ -524,7 +528,7 @@ worker_park (void)
                if (interrupted)
                        goto done;
 
-               if (mono_cond_timedwait_ms (&threadpool->parked_threads_cond, &threadpool->active_threads_lock, rand_next (rand_handle, 5 * 1000, 60 * 1000)) != 0)
+               if (mono_coop_cond_timedwait (&threadpool->parked_threads_cond, &threadpool->active_threads_lock, rand_next ((void **)rand_handle, 5 * 1000, 60 * 1000)) != 0)
                        timeout = TRUE;
 
                mono_thread_info_uninstall_interrupt (&interrupted);
@@ -534,9 +538,7 @@ done:
                threadpool->parked_threads_count -= 1;
        }
 
-       mono_mutex_unlock (&threadpool->active_threads_lock);
-
-       MONO_FINISH_BLOCKING;
+       mono_coop_mutex_unlock (&threadpool->active_threads_lock);
 
        mono_gc_set_skip_thread (FALSE);
 
@@ -552,16 +554,12 @@ worker_try_unpark (void)
 
        mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try unpark worker", mono_native_thread_id_get ());
 
-       MONO_PREPARE_BLOCKING;
-
-       mono_mutex_lock (&threadpool->active_threads_lock);
+       mono_coop_mutex_lock (&threadpool->active_threads_lock);
        if (threadpool->parked_threads_count > 0) {
-               mono_cond_signal (&threadpool->parked_threads_cond);
+               mono_coop_cond_signal (&threadpool->parked_threads_cond);
                res = TRUE;
        }
-       mono_mutex_unlock (&threadpool->active_threads_lock);
-
-       MONO_FINISH_BLOCKING;
+       mono_coop_mutex_unlock (&threadpool->active_threads_lock);
 
        mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try unpark worker, success? %s", mono_native_thread_id_get (), res ? "yes" : "no");
 
@@ -580,6 +578,7 @@ worker_kill (ThreadPoolWorkingThread *thread)
 static void
 worker_thread (gpointer data)
 {
+       MonoError error;
        MonoInternalThread *thread;
        ThreadPoolDomain *tpdomain, *previous_tpdomain;
        ThreadPoolCounter counter;
@@ -594,23 +593,21 @@ worker_thread (gpointer data)
 
        mono_thread_set_name_internal (thread, mono_string_new (mono_domain_get (), "Threadpool worker"), FALSE);
 
-       MONO_PREPARE_BLOCKING;
-       mono_mutex_lock (&threadpool->active_threads_lock);
+       mono_coop_mutex_lock (&threadpool->active_threads_lock);
        g_ptr_array_add (threadpool->working_threads, thread);
-       mono_mutex_unlock (&threadpool->active_threads_lock);
-       MONO_FINISH_BLOCKING;
+       mono_coop_mutex_unlock (&threadpool->active_threads_lock);
 
        previous_tpdomain = NULL;
 
-       mono_mutex_lock (&threadpool->domains_lock);
+       mono_coop_mutex_lock (&threadpool->domains_lock);
 
        while (!mono_runtime_is_shutting_down ()) {
                tpdomain = NULL;
 
                if ((thread->state & (ThreadState_StopRequested | ThreadState_SuspendRequested)) != 0) {
-                       mono_mutex_unlock (&threadpool->domains_lock);
+                       mono_coop_mutex_unlock (&threadpool->domains_lock);
                        mono_thread_interruption_checkpoint ();
-                       mono_mutex_lock (&threadpool->domains_lock);
+                       mono_coop_mutex_lock (&threadpool->domains_lock);
                }
 
                if (retire || !(tpdomain = domain_get_next (previous_tpdomain))) {
@@ -621,9 +618,9 @@ worker_thread (gpointer data)
                                counter._.parked ++;
                        });
 
-                       mono_mutex_unlock (&threadpool->domains_lock);
+                       mono_coop_mutex_unlock (&threadpool->domains_lock);
                        timeout = worker_park ();
-                       mono_mutex_lock (&threadpool->domains_lock);
+                       mono_coop_mutex_lock (&threadpool->domains_lock);
 
                        COUNTER_ATOMIC (counter, {
                                counter._.working ++;
@@ -649,18 +646,23 @@ worker_thread (gpointer data)
                g_assert (tpdomain->domain->threadpool_jobs >= 0);
                tpdomain->domain->threadpool_jobs ++;
 
-               mono_mutex_unlock (&threadpool->domains_lock);
+               mono_coop_mutex_unlock (&threadpool->domains_lock);
 
                mono_thread_push_appdomain_ref (tpdomain->domain);
                if (mono_domain_set (tpdomain->domain, FALSE)) {
-                       MonoObject *exc = NULL;
-                       MonoObject *res = mono_runtime_invoke (mono_defaults.threadpool_perform_wait_callback_method, NULL, NULL, &exc);
-                       if (exc)
+                       MonoObject *exc = NULL, *res;
+
+                       res = mono_runtime_try_invoke (mono_defaults.threadpool_perform_wait_callback_method, NULL, NULL, &exc, &error);
+                       if (exc || !mono_error_ok(&error)) {
+                               if (exc == NULL)
+                                       exc = (MonoObject *) mono_error_convert_to_exception (&error);
+                               else
+                                       mono_error_cleanup (&error);
                                mono_thread_internal_unhandled_exception (exc);
-                       else if (res && *(MonoBoolean*) mono_object_unbox (res) == FALSE)
+                       else if (res && *(MonoBoolean*) mono_object_unbox (res) == FALSE)
                                retire = TRUE;
 
-                       mono_thread_clr_state (thread , ~ThreadState_Background);
+                       mono_thread_clr_state (thread, (MonoThreadState)~ThreadState_Background);
                        if (!mono_thread_test_state (thread , ThreadState_Background))
                                ves_icall_System_Threading_Thread_SetState (thread, ThreadState_Background);
 
@@ -668,7 +670,7 @@ worker_thread (gpointer data)
                }
                mono_thread_pop_appdomain_ref ();
 
-               mono_mutex_lock (&threadpool->domains_lock);
+               mono_coop_mutex_lock (&threadpool->domains_lock);
 
                tpdomain->domain->threadpool_jobs --;
                g_assert (tpdomain->domain->threadpool_jobs >= 0);
@@ -685,13 +687,11 @@ worker_thread (gpointer data)
                previous_tpdomain = tpdomain;
        }
 
-       mono_mutex_unlock (&threadpool->domains_lock);
+       mono_coop_mutex_unlock (&threadpool->domains_lock);
 
-       MONO_PREPARE_BLOCKING;
-       mono_mutex_lock (&threadpool->active_threads_lock);
+       mono_coop_mutex_lock (&threadpool->active_threads_lock);
        g_ptr_array_remove_fast (threadpool->working_threads, thread);
-       mono_mutex_unlock (&threadpool->active_threads_lock);
-       MONO_FINISH_BLOCKING;
+       mono_coop_mutex_unlock (&threadpool->active_threads_lock);
 
        COUNTER_ATOMIC (counter, {
                counter._.working--;
@@ -706,29 +706,56 @@ worker_try_create (void)
 {
        ThreadPoolCounter counter;
        MonoInternalThread *thread;
+       gint32 now;
+
+       mono_coop_mutex_lock (&threadpool->worker_creation_lock);
 
        mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker", mono_native_thread_id_get ());
 
+       if ((now = mono_100ns_ticks () / 10 / 1000 / 1000) == 0) {
+               g_warning ("failed to get 100ns ticks");
+       } else {
+               if (threadpool->worker_creation_current_second != now) {
+                       threadpool->worker_creation_current_second = now;
+                       threadpool->worker_creation_current_count = 0;
+               } else {
+                       g_assert (threadpool->worker_creation_current_count <= WORKER_CREATION_MAX_PER_SEC);
+                       if (threadpool->worker_creation_current_count == WORKER_CREATION_MAX_PER_SEC) {
+                               mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker, failed: maximum number of worker created per second reached, current count = %d",
+                                       mono_native_thread_id_get (), threadpool->worker_creation_current_count);
+                               mono_coop_mutex_unlock (&threadpool->worker_creation_lock);
+                               return FALSE;
+                       }
+               }
+       }
+
        COUNTER_ATOMIC (counter, {
-               if (counter._.working >= counter._.max_working)
+               if (counter._.working >= counter._.max_working) {
+                       mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker, failed: maximum number of working threads reached",
+                               mono_native_thread_id_get ());
+                       mono_coop_mutex_unlock (&threadpool->worker_creation_lock);
                        return FALSE;
+               }
                counter._.working ++;
                counter._.active ++;
        });
 
        if ((thread = mono_thread_create_internal (mono_get_root_domain (), worker_thread, NULL, TRUE, 0)) != NULL) {
-               mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker, created %p",
-                       mono_native_thread_id_get (), thread->tid);
+               threadpool->worker_creation_current_count += 1;
+
+               mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker, created %p, now = %d count = %d", mono_native_thread_id_get (), thread->tid, now, threadpool->worker_creation_current_count);
+               mono_coop_mutex_unlock (&threadpool->worker_creation_lock);
                return TRUE;
        }
 
-       mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker, failed", mono_native_thread_id_get ());
+       mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] try create worker, failed: could not create thread", mono_native_thread_id_get ());
 
        COUNTER_ATOMIC (counter, {
                counter._.working --;
                counter._.active --;
        });
 
+       mono_coop_mutex_unlock (&threadpool->worker_creation_lock);
        return FALSE;
 }
 
@@ -745,11 +772,11 @@ worker_request (MonoDomain *domain)
        if (mono_runtime_is_shutting_down ())
                return FALSE;
 
-       mono_mutex_lock (&threadpool->domains_lock);
+       mono_coop_mutex_lock (&threadpool->domains_lock);
 
        /* synchronize check with worker_thread */
        if (mono_domain_is_unloading (domain)) {
-               mono_mutex_unlock (&threadpool->domains_lock);
+               mono_coop_mutex_unlock (&threadpool->domains_lock);
                return FALSE;
        }
 
@@ -760,7 +787,7 @@ worker_request (MonoDomain *domain)
        mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] request worker, domain = %p, outstanding_request = %d",
                mono_native_thread_id_get (), tpdomain->domain, tpdomain->outstanding_request);
 
-       mono_mutex_unlock (&threadpool->domains_lock);
+       mono_coop_mutex_unlock (&threadpool->domains_lock);
 
        if (threadpool->suspended)
                return FALSE;
@@ -794,8 +821,10 @@ monitor_should_keep_running (void)
                if (mono_runtime_is_shutting_down ()) {
                        should_keep_running = FALSE;
                } else {
+                       mono_coop_mutex_lock (&threadpool->domains_lock);
                        if (!domain_any_has_request ())
                                should_keep_running = FALSE;
+                       mono_coop_mutex_unlock (&threadpool->domains_lock);
 
                        if (!should_keep_running) {
                                if (last_should_keep_running == -1 || mono_100ns_ticks () - last_should_keep_running < MONITOR_MINIMAL_LIFETIME * 1000 * 10) {
@@ -850,8 +879,8 @@ monitor_thread (void)
        mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] monitor thread, started", mono_native_thread_id_get ());
 
        do {
-               MonoInternalThread *thread;
-               gboolean all_waitsleepjoin = TRUE;
+               ThreadPoolCounter counter;
+               gboolean limit_worker_max_reached;
                gint32 interval_left = MONITOR_INTERVAL;
                gint32 awake = 0; /* number of spurious awakes we tolerate before doing a round of rebalancing */
 
@@ -882,43 +911,48 @@ monitor_thread (void)
                if (threadpool->suspended)
                        continue;
 
-               if (mono_runtime_is_shutting_down () || !domain_any_has_request ())
+               if (mono_runtime_is_shutting_down ())
                        continue;
 
-               MONO_PREPARE_BLOCKING;
-               mono_mutex_lock (&threadpool->active_threads_lock);
-               for (i = 0; i < threadpool->working_threads->len; ++i) {
-                       thread = g_ptr_array_index (threadpool->working_threads, i);
-                       if ((thread->state & ThreadState_WaitSleepJoin) == 0) {
-                               all_waitsleepjoin = FALSE;
+               mono_coop_mutex_lock (&threadpool->domains_lock);
+               if (!domain_any_has_request ()) {
+                       mono_coop_mutex_unlock (&threadpool->domains_lock);
+                       continue;
+               }
+               mono_coop_mutex_unlock (&threadpool->domains_lock);
+
+               threadpool->cpu_usage = mono_cpu_usage (threadpool->cpu_usage_state);
+
+               if (!monitor_sufficient_delay_since_last_dequeue ())
+                       continue;
+
+               limit_worker_max_reached = FALSE;
+
+               COUNTER_ATOMIC (counter, {
+                       if (counter._.max_working >= threadpool->limit_worker_max) {
+                               limit_worker_max_reached = TRUE;
                                break;
                        }
-               }
-               mono_mutex_unlock (&threadpool->active_threads_lock);
-               MONO_FINISH_BLOCKING;
+                       counter._.max_working ++;
+               });
 
-               if (all_waitsleepjoin) {
-                       ThreadPoolCounter counter;
-                       COUNTER_ATOMIC (counter, { counter._.max_working ++; });
-                       hill_climbing_force_change (counter._.max_working, TRANSITION_STARVATION);
-               }
+               if (limit_worker_max_reached)
+                       continue;
 
-               threadpool->cpu_usage = mono_cpu_usage (threadpool->cpu_usage_state);
+               hill_climbing_force_change (counter._.max_working, TRANSITION_STARVATION);
 
-               if (monitor_sufficient_delay_since_last_dequeue ()) {
-                       for (i = 0; i < 5; ++i) {
-                               if (mono_runtime_is_shutting_down ())
-                                       break;
+               for (i = 0; i < 5; ++i) {
+                       if (mono_runtime_is_shutting_down ())
+                               break;
 
-                               if (worker_try_unpark ()) {
-                                       mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] monitor thread, unparked", mono_native_thread_id_get ());
-                                       break;
-                               }
+                       if (worker_try_unpark ()) {
+                               mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] monitor thread, unparked", mono_native_thread_id_get ());
+                               break;
+                       }
 
-                               if (worker_try_create ()) {
-                                       mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] monitor thread, created", mono_native_thread_id_get ());
-                                       break;
-                               }
+                       if (worker_try_create ()) {
+                               mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_THREADPOOL, "[%p] monitor thread, created", mono_native_thread_id_get ());
+                               break;
                        }
                }
        } while (monitor_should_keep_running ());
@@ -1249,7 +1283,7 @@ heuristic_adjust (void)
 {
        g_assert (threadpool);
 
-       if (mono_mutex_trylock (&threadpool->heuristic_lock) == 0) {
+       if (mono_coop_mutex_trylock (&threadpool->heuristic_lock) == 0) {
                gint32 completions = InterlockedExchange (&threadpool->heuristic_completions, 0);
                guint32 sample_end = mono_msec_ticks ();
                guint32 sample_duration = sample_end - threadpool->heuristic_sample_start;
@@ -1270,7 +1304,7 @@ heuristic_adjust (void)
                        threadpool->heuristic_last_adjustment = mono_msec_ticks ();
                }
 
-               mono_mutex_unlock (&threadpool->heuristic_lock);
+               mono_coop_mutex_unlock (&threadpool->heuristic_lock);
        }
 }
 
@@ -1284,7 +1318,7 @@ mono_threadpool_ms_cleanup (void)
 }
 
 MonoAsyncResult *
-mono_threadpool_ms_begin_invoke (MonoDomain *domain, MonoObject *target, MonoMethod *method, gpointer *params)
+mono_threadpool_ms_begin_invoke (MonoDomain *domain, MonoObject *target, MonoMethod *method, gpointer *params, MonoError *error)
 {
        static MonoClass *async_call_klass = NULL;
        MonoMethodMessage *message;
@@ -1294,14 +1328,17 @@ mono_threadpool_ms_begin_invoke (MonoDomain *domain, MonoObject *target, MonoMet
        MonoObject *state = NULL;
 
        if (!async_call_klass)
-               async_call_klass = mono_class_from_name (mono_defaults.corlib, "System", "MonoAsyncCall");
-       g_assert (async_call_klass);
+               async_call_klass = mono_class_load_from_name (mono_defaults.corlib, "System", "MonoAsyncCall");
 
        mono_lazy_initialize (&status, initialize);
 
+       mono_error_init (error);
+
        message = mono_method_call_message_new (method, params, mono_get_delegate_invoke (method->klass), (params != NULL) ? (&async_callback) : NULL, (params != NULL) ? (&state) : NULL);
 
-       async_call = (MonoAsyncCall*) mono_object_new (domain, async_call_klass);
+       async_call = (MonoAsyncCall*) mono_object_new_checked (domain, async_call_klass, error);
+       return_val_if_nok (error, NULL);
+
        MONO_OBJECT_SETREF (async_call, msg, message);
        MONO_OBJECT_SETREF (async_call, state, state);
 
@@ -1313,7 +1350,8 @@ mono_threadpool_ms_begin_invoke (MonoDomain *domain, MonoObject *target, MonoMet
        async_result = mono_async_result_new (domain, NULL, async_call->state, NULL, (MonoObject*) async_call);
        MONO_OBJECT_SETREF (async_result, async_delegate, target);
 
-       mono_threadpool_ms_enqueue_work_item (domain, (MonoObject*) async_result);
+       mono_threadpool_ms_enqueue_work_item (domain, (MonoObject*) async_result, error);
+       return_val_if_nok (error, NULL);
 
        return async_result;
 }
@@ -1439,12 +1477,16 @@ mono_threadpool_ms_resume (void)
 void
 ves_icall_System_Threading_ThreadPool_GetAvailableThreadsNative (gint32 *worker_threads, gint32 *completion_port_threads)
 {
+       ThreadPoolCounter counter;
+
        if (!worker_threads || !completion_port_threads)
                return;
 
        mono_lazy_initialize (&status, initialize);
 
-       *worker_threads = threadpool->limit_worker_max;
+       counter.as_gint64 = COUNTER_READ ();
+
+       *worker_threads = MAX (0, threadpool->limit_worker_max - counter._.active);
        *completion_port_threads = threadpool->limit_io_max;
 }
 
@@ -1482,8 +1524,8 @@ ves_icall_System_Threading_ThreadPool_SetMinThreadsNative (gint32 worker_threads
        if (completion_port_threads <= 0 || completion_port_threads > threadpool->limit_io_max)
                return FALSE;
 
-       threadpool->limit_worker_max = worker_threads;
-       threadpool->limit_io_max = completion_port_threads;
+       threadpool->limit_worker_min = worker_threads;
+       threadpool->limit_io_min = completion_port_threads;
 
        return TRUE;
 }
@@ -1547,7 +1589,9 @@ void
 ves_icall_System_Threading_ThreadPool_ReportThreadStatus (MonoBoolean is_working)
 {
        // TODO
-       mono_raise_exception (mono_get_exception_not_implemented (NULL));
+       MonoError error;
+       mono_error_set_not_implemented (&error, "");
+       mono_error_set_pending_exception (&error);
 }
 
 MonoBoolean
@@ -1560,7 +1604,9 @@ MonoBoolean G_GNUC_UNUSED
 ves_icall_System_Threading_ThreadPool_PostQueuedCompletionStatus (MonoNativeOverlapped *native_overlapped)
 {
        /* This copy the behavior of the current Mono implementation */
-       mono_raise_exception (mono_get_exception_not_implemented (NULL));
+       MonoError error;
+       mono_error_set_not_implemented (&error, "");
+       mono_error_set_pending_exception (&error);
        return FALSE;
 }