+#ifdef PARALLEL_MARK
+
+# if defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
+ /* Use pthread-based parallel mark implementation. */
+# define GC_PTHREADS_PARAMARK
+# endif
+
+ /* GC_mark_thread() is the same as in pthread_support.c */
+# ifdef GC_PTHREADS_PARAMARK
+ STATIC void * GC_mark_thread(void * id)
+# else
+# ifdef MSWINCE
+ STATIC DWORD WINAPI GC_mark_thread(LPVOID id)
+# else
+ STATIC unsigned __stdcall GC_mark_thread(void * id)
+# endif
+# endif
+ {
+ word my_mark_no = 0;
+
+ marker_sp[(word)id] = GC_approx_sp();
+# ifdef IA64
+ marker_bsp[(word)id] = GC_save_regs_in_stack();
+# endif
+
+ if ((word)id == (word)-1) return 0; /* to make compiler happy */
+
+ for (;; ++my_mark_no) {
+ if (my_mark_no - GC_mark_no > (word)2) {
+ /* resynchronize if we get far off, e.g. because GC_mark_no */
+ /* wrapped. */
+ my_mark_no = GC_mark_no;
+ }
+# ifdef DEBUG_THREADS
+ GC_printf("Starting mark helper for mark number %lu\n",
+ (unsigned long)my_mark_no);
+# endif
+ GC_help_marker(my_mark_no);
+ }
+ }
+
+# ifdef GC_ASSERTIONS
+ GC_INNER unsigned long GC_mark_lock_holder = NO_THREAD;
+# endif
+
+ /* GC_mark_threads[] is unused here unlike that in pthread_support.c */
+
+# ifdef GC_PTHREADS_PARAMARK
+# include <pthread.h>
+
+# ifndef NUMERIC_THREAD_ID
+# define NUMERIC_THREAD_ID(id) (unsigned long)(id.p)
+# endif
+
+ /* start_mark_threads() is the same as in pthread_support.c except for: */
+ /* - GC_markers value is adjusted already; */
+ /* - thread stack is assumed to be large enough; and */
+ /* - statistics about the number of marker threads is printed outside. */
+ static void start_mark_threads(void)
+ {
+ int i;
+ pthread_attr_t attr;
+ pthread_t new_thread;
+
+ if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
+
+ if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
+ ABORT("pthread_attr_setdetachstate failed");
+
+ for (i = 0; i < GC_markers - 1; ++i) {
+ marker_last_stack_min[i] = ADDR_LIMIT;
+ if (0 != pthread_create(&new_thread, &attr,
+ GC_mark_thread, (void *)(word)i)) {
+ WARN("Marker thread creation failed.\n", 0);
+ /* Don't try to create other marker threads. */
+ GC_markers = i + 1;
+ if (i == 0) GC_parallel = FALSE;
+ break;
+ }
+ }
+ pthread_attr_destroy(&attr);
+ }
+
+ static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+ static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
+
+ /* GC_acquire/release_mark_lock(), GC_wait_builder/marker(), */
+ /* GC_wait_for_reclaim(), GC_notify_all_builder/marker() are the same */
+ /* as in pthread_support.c except that GC_generic_lock() is not used. */
+
+# ifdef LOCK_STATS
+ AO_t GC_block_count = 0;
+# endif
+
+ GC_INNER void GC_acquire_mark_lock(void)
+ {
+ if (pthread_mutex_lock(&mark_mutex) != 0) {
+ ABORT("pthread_mutex_lock failed");
+ }
+# ifdef LOCK_STATS
+ (void)AO_fetch_and_add1(&GC_block_count);
+# endif
+ /* GC_generic_lock(&mark_mutex); */
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
+# endif
+ }
+
+ GC_INNER void GC_release_mark_lock(void)
+ {
+ GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NO_THREAD;
+# endif
+ if (pthread_mutex_unlock(&mark_mutex) != 0) {
+ ABORT("pthread_mutex_unlock failed");
+ }
+ }
+
+ /* Collector must wait for a freelist builders for 2 reasons: */
+ /* 1) Mark bits may still be getting examined without lock. */
+ /* 2) Partial free lists referenced only by locals may not be */
+ /* scanned correctly, e.g. if they contain "pointer-free" objects, */
+ /* since the free-list link may be ignored. */
+ STATIC void GC_wait_builder(void)
+ {
+ GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NO_THREAD;
+# endif
+ if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
+ ABORT("pthread_cond_wait failed");
+ }
+ GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
+# endif
+ }
+
+ GC_INNER void GC_wait_for_reclaim(void)
+ {
+ GC_acquire_mark_lock();
+ while (GC_fl_builder_count > 0) {
+ GC_wait_builder();
+ }
+ GC_release_mark_lock();
+ }
+
+ GC_INNER void GC_notify_all_builder(void)
+ {
+ GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
+ if (pthread_cond_broadcast(&builder_cv) != 0) {
+ ABORT("pthread_cond_broadcast failed");
+ }
+ }
+
+ static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
+
+ GC_INNER void GC_wait_marker(void)
+ {
+ GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NO_THREAD;
+# endif
+ if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
+ ABORT("pthread_cond_wait failed");
+ }
+ GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
+# endif
+ }
+
+ GC_INNER void GC_notify_all_marker(void)
+ {
+ if (pthread_cond_broadcast(&mark_cv) != 0) {
+ ABORT("pthread_cond_broadcast failed");
+ }
+ }
+
+# else /* ! GC_PTHREADS_PARAMARK */
+
+# ifdef DONT_USE_SIGNALANDWAIT
+ STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1] = {0};
+ /* Events with manual reset (one for each */
+ /* mark helper). */
+
+ STATIC DWORD GC_marker_Id[MAX_MARKERS - 1] = {0};
+ /* This table is used for mapping helper */
+ /* threads ID to mark helper index (linear */
+ /* search is used since the mapping contains */
+ /* only a few entries). */
+# endif
+
+# ifndef MARK_THREAD_STACK_SIZE
+# define MARK_THREAD_STACK_SIZE 0 /* default value */
+# endif
+
+ /* mark_mutex_event, builder_cv, mark_cv are initialized in GC_thr_init */
+ static HANDLE mark_mutex_event = (HANDLE)0; /* Event with auto-reset. */
+ static HANDLE builder_cv = (HANDLE)0; /* Event with manual reset. */
+ static HANDLE mark_cv = (HANDLE)0; /* Event with manual reset. */
+
+ static void start_mark_threads(void)
+ {
+ int i;
+# ifdef MSWINCE
+ HANDLE handle;
+ DWORD thread_id;
+# else
+ GC_uintptr_t handle;
+ unsigned thread_id;
+# endif
+
+# ifdef DONT_USE_SIGNALANDWAIT
+ /* Initialize GC_marker_cv[] and GC_marker_Id[] fully before */
+ /* starting the first helper thread. */
+ for (i = 0; i < GC_markers - 1; ++i) {
+ GC_marker_Id[i] = GetCurrentThreadId();
+ if ((GC_marker_cv[i] = CreateEvent(NULL /* attrs */,
+ TRUE /* isManualReset */,
+ FALSE /* initialState */,
+ NULL /* name (A/W) */)) == (HANDLE)0)
+ ABORT("CreateEvent() failed");
+ }
+# endif
+
+ for (i = 0; i < GC_markers - 1; ++i) {
+ marker_last_stack_min[i] = ADDR_LIMIT;
+# ifdef MSWINCE
+ /* There is no _beginthreadex() in WinCE. */
+ handle = CreateThread(NULL /* lpsa */,
+ MARK_THREAD_STACK_SIZE /* ignored */,
+ GC_mark_thread, (LPVOID)(word)i,
+ 0 /* fdwCreate */, &thread_id);
+ if (handle == NULL) {
+ WARN("Marker thread creation failed\n", 0);
+ /* The most probable failure reason is "not enough memory". */
+ /* Don't try to create other marker threads. */
+ break;
+ } else {
+ /* It's safe to detach the thread. */
+ CloseHandle(handle);
+ }
+# else
+ handle = _beginthreadex(NULL /* security_attr */,
+ MARK_THREAD_STACK_SIZE, GC_mark_thread,
+ (void *)(word)i, 0 /* flags */, &thread_id);
+ if (!handle || handle == (GC_uintptr_t)-1L) {
+ WARN("Marker thread creation failed\n", 0);
+ /* Don't try to create other marker threads. */
+ break;
+ } else {/* We may detach the thread (if handle is of HANDLE type) */
+ /* CloseHandle((HANDLE)handle); */
+ }
+# endif
+ }
+
+ /* Adjust GC_markers (and free unused resources) in case of failure. */
+# ifdef DONT_USE_SIGNALANDWAIT
+ while ((int)GC_markers > i + 1) {
+ GC_markers--;
+ CloseHandle(GC_marker_cv[(int)GC_markers - 1]);
+ }
+# else
+ GC_markers = i + 1;
+# endif
+ if (i == 0) {
+ GC_parallel = FALSE;
+ CloseHandle(mark_cv);
+ CloseHandle(builder_cv);
+ CloseHandle(mark_mutex_event);
+ }
+ }
+
+# ifdef DONT_USE_SIGNALANDWAIT
+ STATIC /* volatile */ LONG GC_mark_mutex_state = 0;
+ /* Mutex state: 0 - unlocked, */
+ /* 1 - locked and no other waiters, */
+ /* -1 - locked and waiters may exist. */
+ /* Accessed by InterlockedExchange(). */
+# else
+ STATIC volatile AO_t GC_mark_mutex_waitcnt = 0;
+ /* Number of waiters + 1; 0 - unlocked. */
+# endif
+
+ /* #define LOCK_STATS */
+# ifdef LOCK_STATS
+ AO_t GC_block_count = 0;
+ AO_t GC_unlocked_count = 0;
+# endif
+
+ GC_INNER void GC_acquire_mark_lock(void)
+ {
+# ifdef DONT_USE_SIGNALANDWAIT
+ if (InterlockedExchange(&GC_mark_mutex_state, 1 /* locked */) != 0)
+# else
+ if (AO_fetch_and_add1_acquire(&GC_mark_mutex_waitcnt) != 0)
+# endif
+ {
+# ifdef LOCK_STATS
+ (void)AO_fetch_and_add1(&GC_block_count);
+# endif
+# ifdef DONT_USE_SIGNALANDWAIT
+ /* Repeatedly reset the state and wait until acquire the lock. */
+ while (InterlockedExchange(&GC_mark_mutex_state,
+ -1 /* locked_and_has_waiters */) != 0)
+# endif
+ {
+ if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
+ ABORT("WaitForSingleObject() failed");
+ }
+ }
+# ifdef LOCK_STATS
+ else {
+ (void)AO_fetch_and_add1(&GC_unlocked_count);
+ }
+# endif
+
+ GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = (unsigned long)GetCurrentThreadId();
+# endif
+ }
+
+ GC_INNER void GC_release_mark_lock(void)
+ {
+ GC_ASSERT(GC_mark_lock_holder == (unsigned long)GetCurrentThreadId());
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NO_THREAD;
+# endif
+# ifdef DONT_USE_SIGNALANDWAIT
+ if (InterlockedExchange(&GC_mark_mutex_state, 0 /* unlocked */) < 0)
+# else
+ GC_ASSERT(AO_load(&GC_mark_mutex_waitcnt) != 0);
+ if (AO_fetch_and_sub1_release(&GC_mark_mutex_waitcnt) > 1)
+# endif
+ {
+ /* wake a waiter */
+ if (SetEvent(mark_mutex_event) == FALSE)
+ ABORT("SetEvent() failed");
+ }
+ }
+
+ /* In GC_wait_for_reclaim/GC_notify_all_builder() we emulate POSIX */
+ /* cond_wait/cond_broadcast() primitives with WinAPI Event object */
+ /* (working in "manual reset" mode). This works here because */
+ /* GC_notify_all_builder() is always called holding lock on */
+ /* mark_mutex and the checked condition (GC_fl_builder_count == 0) */
+ /* is the only one for which broadcasting on builder_cv is performed. */
+
+ GC_INNER void GC_wait_for_reclaim(void)
+ {
+ GC_ASSERT(builder_cv != 0);
+ for (;;) {
+ GC_acquire_mark_lock();
+ if (GC_fl_builder_count == 0)
+ break;
+ if (ResetEvent(builder_cv) == FALSE)
+ ABORT("ResetEvent() failed");
+ GC_release_mark_lock();
+ if (WaitForSingleObject(builder_cv, INFINITE) == WAIT_FAILED)
+ ABORT("WaitForSingleObject() failed");
+ }
+ GC_release_mark_lock();
+ }
+
+ GC_INNER void GC_notify_all_builder(void)
+ {
+ GC_ASSERT(GC_mark_lock_holder == (unsigned long)GetCurrentThreadId());
+ GC_ASSERT(builder_cv != 0);
+ GC_ASSERT(GC_fl_builder_count == 0);
+ if (SetEvent(builder_cv) == FALSE)
+ ABORT("SetEvent() failed");
+ }
+
+# ifdef DONT_USE_SIGNALANDWAIT
+
+ /* mark_cv is used (for waiting) by a non-helper thread. */
+
+ GC_INNER void GC_wait_marker(void)
+ {
+ HANDLE event = mark_cv;
+ DWORD id = GetCurrentThreadId();
+ int i = (int)GC_markers - 1;
+ while (i-- > 0) {
+ if (GC_marker_Id[i] == id) {
+ event = GC_marker_cv[i];
+ break;
+ }
+ }
+
+ if (ResetEvent(event) == FALSE)
+ ABORT("ResetEvent() failed");
+ GC_release_mark_lock();
+ if (WaitForSingleObject(event, INFINITE) == WAIT_FAILED)
+ ABORT("WaitForSingleObject() failed");
+ GC_acquire_mark_lock();
+ }
+
+ GC_INNER void GC_notify_all_marker(void)
+ {
+ DWORD id = GetCurrentThreadId();
+ int i = (int)GC_markers - 1;
+ while (i-- > 0) {
+ /* Notify every marker ignoring self (for efficiency). */
+ if (SetEvent(GC_marker_Id[i] != id ? GC_marker_cv[i] :
+ mark_cv) == FALSE)
+ ABORT("SetEvent() failed");
+ }
+ }
+
+# else /* DONT_USE_SIGNALANDWAIT */
+
+ /* For GC_wait_marker/GC_notify_all_marker() the above technique */
+ /* does not work because they are used with different checked */
+ /* conditions in different places (and, in addition, notifying is */
+ /* done after leaving critical section) and this could result in */
+ /* a signal loosing between checking for a particular condition */
+ /* and calling WaitForSingleObject. So, we use PulseEvent() and */
+ /* NT SignalObjectAndWait() (which atomically sets mutex event to */
+ /* signaled state and starts waiting on condvar). A special */
+ /* case here is GC_mark_mutex_waitcnt == 1 (i.e. nobody waits for */
+ /* mark lock at this moment) - we don't change it (otherwise we */
+ /* may loose a signal sent between decrementing */
+ /* GC_mark_mutex_waitcnt and calling WaitForSingleObject()). */
+
+# ifdef MSWINCE
+ /* SignalObjectAndWait() is missing in WinCE (for now), so you */
+ /* should supply its emulation (externally) to use this code. */
+ WINBASEAPI DWORD WINAPI SignalObjectAndWait(HANDLE, HANDLE, DWORD,
+ BOOL);
+# define signalObjectAndWait_func SignalObjectAndWait
+# else
+ typedef DWORD (WINAPI * SignalObjectAndWait_type)(HANDLE, HANDLE,
+ DWORD, BOOL);
+ static SignalObjectAndWait_type signalObjectAndWait_func = 0;
+# endif
+
+ GC_INNER void GC_wait_marker(void)
+ {
+ /* Here we assume that GC_wait_marker() is always called */
+ /* from a while(check_cond) loop. */
+ AO_t waitcnt;
+ GC_ASSERT(mark_cv != 0);
+
+ /* We inline GC_release_mark_lock() to have atomic */
+ /* unlock-and-wait action here. */
+ GC_ASSERT(GC_mark_lock_holder == (unsigned long)GetCurrentThreadId());
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = NO_THREAD;
+# endif
+
+ if ((waitcnt = AO_load(&GC_mark_mutex_waitcnt)) > 1) {
+ (void)AO_fetch_and_sub1_release(&GC_mark_mutex_waitcnt);
+ } else {
+ GC_ASSERT(AO_load(&GC_mark_mutex_waitcnt) != 0);
+ }
+
+ /* The state of mark_cv is non-signaled here. */
+ if (signalObjectAndWait_func(mark_mutex_event /* hObjectToSignal */,
+ mark_cv /* hObjectToWaitOn */,
+ INFINITE /* timeout */,
+ FALSE /* isAlertable */) == WAIT_FAILED)
+ ABORT("SignalObjectAndWait() failed");
+ /* The state of mark_cv is non-signaled here again. */
+
+ if (waitcnt > 1) {
+ GC_acquire_mark_lock();
+ } else {
+ GC_ASSERT(GC_mark_mutex_waitcnt != 0);
+ /* Acquire mark lock */
+ if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
+ ABORT("WaitForSingleObject() failed");
+ GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
+# ifdef GC_ASSERTIONS
+ GC_mark_lock_holder = (unsigned long)GetCurrentThreadId();
+# endif
+ }
+ }
+
+ GC_INNER void GC_notify_all_marker(void)
+ {
+ GC_ASSERT(mark_cv != 0);
+ if (PulseEvent(mark_cv) == FALSE)
+ ABORT("PulseEvent() failed");
+ }
+
+# endif /* !DONT_USE_SIGNALANDWAIT */
+
+# endif /* ! GC_PTHREADS_PARAMARK */
+
+#endif /* PARALLEL_MARK */
+