2 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
3 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
4 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
5 * Copyright (c) 2000-2008 by Hewlett-Packard Development Company.
8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK.
11 * Permission is hereby granted to use or copy this program
12 * for any purpose, provided the above notices are retained on all copies.
13 * Permission to modify the code and to distribute modified code is granted,
14 * provided the above notices are retained, and a notice that the code was
15 * modified is included with the above copyright notice.
18 #include "private/gc_priv.h"
20 #if defined(GC_WIN32_THREADS)
22 #ifndef WIN32_LEAN_AND_MEAN
23 # define WIN32_LEAN_AND_MEAN 1
28 #ifdef THREAD_LOCAL_ALLOC
29 # include "private/thread_local_alloc.h"
30 #endif /* THREAD_LOCAL_ALLOC */
32 /* Allocation lock declarations. */
33 #if !defined(USE_PTHREAD_LOCKS)
34 GC_INNER CRITICAL_SECTION GC_allocate_ml;
35 GC_INNER DWORD GC_lock_holder = NO_THREAD;
36 /* Thread id for current holder of allocation lock */
38 GC_INNER pthread_mutex_t GC_allocate_ml = PTHREAD_MUTEX_INITIALIZER;
39 GC_INNER unsigned long GC_lock_holder = NO_THREAD;
43 # include <errno.h> /* for EAGAIN */
45 /* Cygwin-specific forward decls */
46 # undef pthread_create
48 # undef pthread_detach
50 # ifndef GC_NO_PTHREAD_SIGMASK
51 # undef pthread_sigmask
54 STATIC void * GC_pthread_start(void * arg);
55 STATIC void GC_thread_exit_proc(void *arg);
58 # ifdef CAN_HANDLE_FORK
66 # undef _beginthreadex
70 /* Force DONT_USE_SIGNALANDWAIT implementation of PARALLEL_MARK */
71 /* for WinCE (since Win32 SignalObjectAndWait() is missing). */
72 # ifndef DONT_USE_SIGNALANDWAIT
73 # define DONT_USE_SIGNALANDWAIT
76 # include <process.h> /* For _beginthreadex, _endthreadex */
77 # include <errno.h> /* for errno, EAGAIN */
82 /* DllMain-based thread registration is currently incompatible */
83 /* with thread-local allocation, pthreads and WinCE. */
84 #if defined(GC_DLL) && !defined(GC_NO_THREADS_DISCOVERY) && !defined(MSWINCE) \
85 && !defined(THREAD_LOCAL_ALLOC) && !defined(GC_PTHREADS)
86 # include "atomic_ops.h"
88 /* This code operates in two distinct modes, depending on */
89 /* the setting of GC_win32_dll_threads. */
90 /* If GC_win32_dll_threads is set, all threads in the process */
91 /* are implicitly registered with the GC by DllMain. */
92 /* No explicit registration is required, and attempts at */
93 /* explicit registration are ignored. This mode is */
94 /* very different from the Posix operation of the collector. */
95 /* In this mode access to the thread table is lock-free. */
96 /* Hence there is a static limit on the number of threads. */
98 # ifdef GC_DISCOVER_TASK_THREADS
99 /* GC_DISCOVER_TASK_THREADS should be used if DllMain-based */
100 /* thread registration is required but it is impossible to */
101 /* call GC_use_threads_discovery before other GC routines. */
102 # define GC_win32_dll_threads TRUE
104 STATIC GC_bool GC_win32_dll_threads = FALSE;
105 /* GC_win32_dll_threads must be set (if needed) at the */
106 /* application initialization time, i.e. before any */
107 /* collector or thread calls. We make it a "dynamic" */
108 /* option only to avoid multiple library versions. */
112 /* If GC_win32_dll_threads is FALSE (or the collector is */
113 /* built without GC_DLL defined), things operate in a way */
114 /* that is very similar to Posix platforms, and new threads */
115 /* must be registered with the collector, e.g. by using */
116 /* preprocessor-based interception of the thread primitives. */
117 /* In this case, we use a real data structure for the thread */
118 /* table. Note that there is no equivalent of linker-based */
119 /* call interception, since we don't have ELF-like */
120 /* facilities. The Windows analog appears to be "API */
121 /* hooking", which really seems to be a standard way to */
122 /* do minor binary rewriting (?). I'd prefer not to have */
123 /* the basic collector rely on such facilities, but an */
124 /* optional package that intercepts thread calls this way */
125 /* would probably be nice. */
126 # ifndef GC_NO_THREADS_DISCOVERY
127 # define GC_NO_THREADS_DISCOVERY
129 # define GC_win32_dll_threads FALSE
131 # define MAX_THREADS 1 /* dll_thread_table[] is always empty. */
132 #endif /* GC_NO_THREADS_DISCOVERY */
134 /* We have two versions of the thread table. Which one */
135 /* we us depends on whether or not GC_win32_dll_threads */
136 /* is set. Note that before initialization, we don't */
137 /* add any entries to either table, even if DllMain is */
138 /* called. The main thread will be added on */
139 /* initialization. */
141 /* The type of the first argument to InterlockedExchange. */
142 /* Documented to be LONG volatile *, but at least gcc likes */
146 STATIC GC_bool GC_thr_initialized = FALSE;
148 GC_INNER GC_bool GC_need_to_lock = FALSE;
150 static GC_bool parallel_initialized = FALSE;
152 /* GC_use_threads_discovery() is currently incompatible with pthreads */
153 /* and WinCE. It might be possible to get DllMain-based thread */
154 /* registration to work with Cygwin, but if you try it then you are on */
156 GC_API void GC_CALL GC_use_threads_discovery(void)
158 # ifdef GC_NO_THREADS_DISCOVERY
159 ABORT("GC DllMain-based thread registration unsupported");
161 /* Turn on GC_win32_dll_threads. */
162 GC_ASSERT(!parallel_initialized);
163 # ifndef GC_DISCOVER_TASK_THREADS
164 GC_win32_dll_threads = TRUE;
170 STATIC DWORD GC_main_thread = 0;
172 #define ADDR_LIMIT ((ptr_t)(word)-1)
174 struct GC_Thread_Rep {
176 # ifndef GC_NO_THREADS_DISCOVERY
177 AO_t in_use; /* Updated without lock. */
178 /* We assert that unused */
179 /* entries have invalid ids of */
180 /* zero and zero stack fields. */
181 /* Used only with GC_win32_dll_threads. */
183 struct GC_Thread_Rep * next;
184 /* Hash table link without */
185 /* GC_win32_dll_threads. */
186 /* More recently allocated threads */
187 /* with a given pthread id come */
188 /* first. (All but the first are */
189 /* guaranteed to be dead, but we may */
190 /* not yet have registered the join.) */
191 } tm; /* table_management */
195 /* According to MSDN specs for WinCE targets: */
196 /* - DuplicateHandle() is not applicable to thread handles; and */
197 /* - the value returned by GetCurrentThreadId() could be used as */
198 /* a "real" thread handle (for SuspendThread(), ResumeThread() and */
199 /* GetThreadContext()). */
200 # define THREAD_HANDLE(t) (HANDLE)(word)(t)->id
203 # define THREAD_HANDLE(t) (t)->handle
206 ptr_t stack_base; /* The cold end of the stack. */
207 /* 0 ==> entry not valid. */
208 /* !in_use ==> stack_base == 0 */
209 ptr_t last_stack_min; /* Last known minimum (hottest) address */
210 /* in stack or ADDR_LIMIT if unset */
212 ptr_t backing_store_end;
213 ptr_t backing_store_ptr;
216 ptr_t thread_blocked_sp; /* Protected by GC lock. */
217 /* NULL value means thread unblocked. */
218 /* If set to non-NULL, thread will */
219 /* acquire GC lock before doing any */
220 /* pointer manipulations. Thus it does */
221 /* not need to stop this thread. */
223 struct GC_traced_stack_sect_s *traced_stack_sect;
224 /* Points to the "stack section" data */
225 /* held in stack by the innermost */
226 /* GC_call_with_gc_active() of this */
227 /* thread. May be NULL. */
229 unsigned short finalizer_skipped;
230 unsigned char finalizer_nested;
231 /* Used by GC_check_finalizer_nested() */
232 /* to minimize the level of recursion */
233 /* when a client finalizer allocates */
234 /* memory (initially both are 0). */
236 unsigned char suspended; /* really of GC_bool type */
239 unsigned char flags; /* Protected by GC lock. */
240 # define FINISHED 1 /* Thread has exited. */
241 # define DETACHED 2 /* Thread is intended to be detached. */
242 # define KNOWN_FINISHED(t) (((t) -> flags) & FINISHED)
243 pthread_t pthread_id;
244 void *status; /* hold exit value until join in case it's a pointer */
246 # define KNOWN_FINISHED(t) 0
249 # ifdef THREAD_LOCAL_ALLOC
250 struct thread_local_freelists tlfs;
254 typedef struct GC_Thread_Rep * GC_thread;
255 typedef volatile struct GC_Thread_Rep * GC_vthread;
257 #ifndef GC_NO_THREADS_DISCOVERY
258 /* We assumed that volatile ==> memory ordering, at least among */
259 /* volatiles. This code should consistently use atomic_ops. */
260 STATIC volatile GC_bool GC_please_stop = FALSE;
261 #elif defined(GC_ASSERTIONS)
262 STATIC GC_bool GC_please_stop = FALSE;
266 * We track thread attachments while the world is supposed to be stopped.
267 * Unfortunately, we can't stop them from starting, since blocking in
268 * DllMain seems to cause the world to deadlock. Thus we have to recover
269 * If we notice this in the middle of marking.
272 #ifndef GC_NO_THREADS_DISCOVERY
273 STATIC AO_t GC_attached_thread = FALSE;
276 #if !defined(__GNUC__)
277 /* Return TRUE if an thread was attached since we last asked or */
278 /* since GC_attached_thread was explicitly reset. */
279 GC_bool GC_started_thread_while_stopped(void)
281 # ifndef GC_NO_THREADS_DISCOVERY
284 if (GC_win32_dll_threads) {
285 AO_nop_full(); /* Prior heap reads need to complete earlier. */
286 result = AO_load(&GC_attached_thread);
288 AO_store(&GC_attached_thread, FALSE);
295 #endif /* !__GNUC__ */
297 /* Thread table used if GC_win32_dll_threads is set. */
298 /* This is a fixed size array. */
299 /* Since we use runtime conditionals, both versions */
300 /* are always defined. */
302 # define MAX_THREADS 512
305 /* Things may get quite slow for large numbers of threads, */
306 /* since we look them up with sequential search. */
307 volatile struct GC_Thread_Rep dll_thread_table[MAX_THREADS];
309 STATIC volatile LONG GC_max_thread_index = 0;
310 /* Largest index in dll_thread_table */
311 /* that was ever used. */
313 /* And now the version used if GC_win32_dll_threads is not set. */
314 /* This is a chained hash table, with much of the code borrowed */
315 /* From the Posix implementation. */
316 #ifndef THREAD_TABLE_SZ
317 # define THREAD_TABLE_SZ 256 /* Power of 2 (for speed). */
319 #define THREAD_TABLE_INDEX(id) (((word)(id) >> 2) % THREAD_TABLE_SZ)
320 STATIC GC_thread GC_threads[THREAD_TABLE_SZ];
322 /* It may not be safe to allocate when we register the first thread. */
323 /* Thus we allocated one statically. */
324 static struct GC_Thread_Rep first_thread;
325 static GC_bool first_thread_used = FALSE;
327 /* Add a thread to GC_threads. We assume it wasn't already there. */
328 /* Caller holds allocation lock. */
329 /* Unlike the pthreads version, the id field is set by the caller. */
330 STATIC GC_thread GC_new_thread(DWORD id)
332 word hv = THREAD_TABLE_INDEX(id);
335 GC_ASSERT(I_HOLD_LOCK());
336 if (!first_thread_used) {
337 result = &first_thread;
338 first_thread_used = TRUE;
340 GC_ASSERT(!GC_win32_dll_threads);
341 result = (struct GC_Thread_Rep *)
342 GC_INTERNAL_MALLOC(sizeof(struct GC_Thread_Rep), NORMAL);
343 /* result can be NULL */
344 if (result == 0) return(0);
346 /* result -> id = id; Done by caller. */
347 result -> tm.next = GC_threads[hv];
348 GC_threads[hv] = result;
350 GC_ASSERT(result -> flags == 0);
352 GC_ASSERT(result -> thread_blocked_sp == NULL);
356 STATIC GC_bool GC_in_thread_creation = FALSE;
357 /* Protected by allocation lock. */
359 GC_INLINE void GC_record_stack_base(GC_vthread me,
360 const struct GC_stack_base *sb)
362 me -> stack_base = sb -> mem_base;
364 me -> backing_store_end = sb -> reg_base;
366 if (me -> stack_base == NULL)
367 ABORT("Bad stack base in GC_register_my_thread");
370 /* This may be called from DllMain, and hence operates under unusual */
371 /* constraints. In particular, it must be lock-free if */
372 /* GC_win32_dll_threads is set. Always called from the thread being */
373 /* added. If GC_win32_dll_threads is not set, we already hold the */
374 /* allocation lock except possibly during single-threaded startup code. */
375 STATIC GC_thread GC_register_my_thread_inner(const struct GC_stack_base *sb,
380 /* The following should be a no-op according to the win32 */
381 /* documentation. There is empirical evidence that it */
383 # if defined(MPROTECT_VDB)
386 && !GC_gww_dirty_init()
389 GC_set_write_fault_handler();
392 # ifndef GC_NO_THREADS_DISCOVERY
393 if (GC_win32_dll_threads) {
395 /* It appears to be unsafe to acquire a lock here, since this */
396 /* code is apparently not preemptible on some systems. */
397 /* (This is based on complaints, not on Microsoft's official */
398 /* documentation, which says this should perform "only simple */
399 /* initialization tasks".) */
400 /* Hence we make do with nonblocking synchronization. */
401 /* It has been claimed that DllMain is really only executed with */
402 /* a particular system lock held, and thus careful use of locking */
403 /* around code that doesn't call back into the system libraries */
404 /* might be OK. But this hasn't been tested across all win32 */
406 /* cast away volatile qualifier */
408 InterlockedExchange((void*)&dll_thread_table[i].tm.in_use, 1) != 0;
410 /* Compare-and-swap would make this cleaner, but that's not */
411 /* supported before Windows 98 and NT 4.0. In Windows 2000, */
412 /* InterlockedExchange is supposed to be replaced by */
413 /* InterlockedExchangePointer, but that's not really what I */
415 /* FIXME: We should eventually declare Win95 dead and use AO_ */
416 /* primitives here. */
417 if (i == MAX_THREADS - 1)
418 ABORT("Too many threads");
420 /* Update GC_max_thread_index if necessary. The following is */
421 /* safe, and unlike CompareExchange-based solutions seems to work */
422 /* on all Windows95 and later platforms. */
423 /* Unfortunately, GC_max_thread_index may be temporarily out of */
424 /* bounds, so readers have to compensate. */
425 while (i > GC_max_thread_index) {
426 InterlockedIncrement((IE_t)&GC_max_thread_index);
428 if (GC_max_thread_index >= MAX_THREADS) {
429 /* We overshot due to simultaneous increments. */
430 /* Setting it to MAX_THREADS-1 is always safe. */
431 GC_max_thread_index = MAX_THREADS - 1;
433 me = dll_thread_table + i;
436 /* else */ /* Not using DllMain */ {
437 GC_ASSERT(I_HOLD_LOCK());
438 GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */
439 me = GC_new_thread(thread_id);
440 GC_in_thread_creation = FALSE;
442 ABORT("Failed to allocate memory for thread registering");
445 /* me can be NULL -> segfault */
446 me -> pthread_id = pthread_self();
449 /* GetCurrentThread() returns a pseudohandle (a const value). */
450 if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
452 (HANDLE*)&(me -> handle),
453 0 /* dwDesiredAccess */, FALSE /* bInheritHandle */,
454 DUPLICATE_SAME_ACCESS)) {
456 GC_log_printf("DuplicateHandle failed with error code: %d\n",
457 (int)GetLastError());
458 ABORT("DuplicateHandle failed");
461 me -> last_stack_min = ADDR_LIMIT;
462 GC_record_stack_base(me, sb);
463 /* Up until this point, GC_push_all_stacks considers this thread */
465 /* Up until this point, this entry is viewed as reserved but invalid */
466 /* by GC_delete_thread. */
467 me -> id = thread_id;
468 # if defined(THREAD_LOCAL_ALLOC)
469 GC_init_thread_local((GC_tlfs)(&(me->tlfs)));
471 # ifndef GC_NO_THREADS_DISCOVERY
472 if (GC_win32_dll_threads) {
473 if (GC_please_stop) {
474 AO_store(&GC_attached_thread, TRUE);
475 AO_nop_full(); /* Later updates must become visible after this. */
477 /* We'd like to wait here, but can't, since waiting in DllMain */
478 /* provokes deadlocks. */
479 /* Thus we force marking to be restarted instead. */
483 GC_ASSERT(!GC_please_stop);
484 /* Otherwise both we and the thread stopping code would be */
485 /* holding the allocation lock. */
487 return (GC_thread)(me);
491 * GC_max_thread_index may temporarily be larger than MAX_THREADS.
492 * To avoid subscript errors, we check on access.
494 GC_INLINE LONG GC_get_max_thread_index(void)
496 LONG my_max = GC_max_thread_index;
497 if (my_max >= MAX_THREADS) return MAX_THREADS - 1;
501 /* Return the GC_thread corresponding to a thread id. May be called */
502 /* without a lock, but should be called in contexts in which the */
503 /* requested thread cannot be asynchronously deleted, e.g. from the */
505 /* This version assumes that either GC_win32_dll_threads is set, or */
506 /* we hold the allocator lock. */
507 /* Also used (for assertion checking only) from thread_local_alloc.c. */
508 STATIC GC_thread GC_lookup_thread_inner(DWORD thread_id)
510 # ifndef GC_NO_THREADS_DISCOVERY
511 if (GC_win32_dll_threads) {
513 LONG my_max = GC_get_max_thread_index();
514 for (i = 0; i <= my_max &&
515 (!AO_load_acquire(&dll_thread_table[i].tm.in_use)
516 || dll_thread_table[i].id != thread_id);
517 /* Must still be in_use, since nobody else can store our */
522 return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
526 word hv = THREAD_TABLE_INDEX(thread_id);
527 register GC_thread p = GC_threads[hv];
529 GC_ASSERT(I_HOLD_LOCK());
530 while (p != 0 && p -> id != thread_id) p = p -> tm.next;
536 # define CHECK_LOOKUP_MY_THREAD(me) \
537 if (!(me)) ABORT("GC_lookup_thread_inner(GetCurrentThreadId) failed")
539 # define CHECK_LOOKUP_MY_THREAD(me) /* empty */
542 /* Called by GC_finalize() (in case of an allocation failure observed). */
543 /* GC_reset_finalizer_nested() is the same as in pthread_support.c. */
544 GC_INNER void GC_reset_finalizer_nested(void)
546 GC_thread me = GC_lookup_thread_inner(GetCurrentThreadId());
547 CHECK_LOOKUP_MY_THREAD(me);
548 me->finalizer_nested = 0;
551 /* Checks and updates the thread-local level of finalizers recursion. */
552 /* Returns NULL if GC_invoke_finalizers() should not be called by the */
553 /* collector (to minimize the risk of a deep finalizers recursion), */
554 /* otherwise returns a pointer to the thread-local finalizer_nested. */
555 /* Called by GC_notify_or_invoke_finalizers() only (the lock is held). */
556 /* GC_check_finalizer_nested() is the same as in pthread_support.c. */
557 GC_INNER unsigned char *GC_check_finalizer_nested(void)
559 GC_thread me = GC_lookup_thread_inner(GetCurrentThreadId());
560 unsigned nesting_level;
561 CHECK_LOOKUP_MY_THREAD(me);
562 nesting_level = me->finalizer_nested;
564 /* We are inside another GC_invoke_finalizers(). */
565 /* Skip some implicitly-called GC_invoke_finalizers() */
566 /* depending on the nesting (recursion) level. */
567 if (++me->finalizer_skipped < (1U << nesting_level)) return NULL;
568 me->finalizer_skipped = 0;
570 me->finalizer_nested = (unsigned char)(nesting_level + 1);
571 return &me->finalizer_nested;
574 #if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)
575 /* This is called from thread-local GC_malloc(). */
576 GC_bool GC_is_thread_tsd_valid(void *tsd)
582 me = GC_lookup_thread_inner(GetCurrentThreadId());
584 return (char *)tsd >= (char *)&me->tlfs
585 && (char *)tsd < (char *)&me->tlfs + sizeof(me->tlfs);
587 #endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */
589 /* Make sure thread descriptor t is not protected by the VDB */
590 /* implementation. */
591 /* Used to prevent write faults when the world is (partially) stopped, */
592 /* since it may have been stopped with a system lock held, and that */
593 /* lock may be required for fault handling. */
594 #if defined(MPROTECT_VDB)
595 # define UNPROTECT_THREAD(t) \
596 if (!GC_win32_dll_threads && GC_dirty_maintained \
597 && t != &first_thread) { \
598 GC_ASSERT(SMALL_OBJ(GC_size(t))); \
599 GC_remove_protection(HBLKPTR(t), 1, FALSE); \
602 # define UNPROTECT_THREAD(t)
606 # define GC_PTHREAD_PTRVAL(pthread_id) pthread_id
607 #elif defined(GC_WIN32_PTHREADS)
608 # define GC_PTHREAD_PTRVAL(pthread_id) pthread_id.p
611 /* If a thread has been joined, but we have not yet */
612 /* been notified, then there may be more than one thread */
613 /* in the table with the same win32 id. */
614 /* This is OK, but we need a way to delete a specific one. */
615 /* Assumes we hold the allocation lock unless */
616 /* GC_win32_dll_threads is set. */
617 /* If GC_win32_dll_threads is set it should be called from the */
618 /* thread being deleted. */
619 STATIC void GC_delete_gc_thread(GC_vthread t)
622 CloseHandle(t->handle);
624 # ifndef GC_NO_THREADS_DISCOVERY
625 if (GC_win32_dll_threads) {
626 /* This is intended to be lock-free. */
627 /* It is either called synchronously from the thread being */
628 /* deleted, or by the joining thread. */
629 /* In this branch asynchronous changes to (*t) are possible. */
630 /* It's not allowed to call GC_printf (and the friends) here, */
631 /* see GC_stop_world() for the information. */
635 GC_PTHREAD_PTRVAL(t->pthread_id) = 0;
637 AO_store_release(&t->tm.in_use, FALSE);
641 DWORD id = ((GC_thread)t) -> id;
642 /* Cast away volatile qualifier, since we have lock. */
643 word hv = THREAD_TABLE_INDEX(id);
644 register GC_thread p = GC_threads[hv];
645 register GC_thread prev = 0;
647 GC_ASSERT(I_HOLD_LOCK());
648 while (p != (GC_thread)t) {
653 GC_threads[hv] = p -> tm.next;
655 prev -> tm.next = p -> tm.next;
661 /* Delete a thread from GC_threads. We assume it is there. */
662 /* (The code intentionally traps if it wasn't.) Assumes we */
663 /* hold the allocation lock unless GC_win32_dll_threads is set. */
664 /* If GC_win32_dll_threads is set then it should be called from */
665 /* the thread being deleted. It is also safe to delete the */
666 /* main thread (unless GC_win32_dll_threads). */
667 STATIC void GC_delete_thread(DWORD id)
669 if (GC_win32_dll_threads) {
670 GC_thread t = GC_lookup_thread_inner(id);
673 WARN("Removing nonexistent thread, id = %" GC_PRIdPTR "\n", id);
675 GC_delete_gc_thread(t);
678 word hv = THREAD_TABLE_INDEX(id);
679 register GC_thread p = GC_threads[hv];
680 register GC_thread prev = 0;
682 GC_ASSERT(I_HOLD_LOCK());
683 while (p -> id != id) {
688 CloseHandle(p->handle);
691 GC_threads[hv] = p -> tm.next;
693 prev -> tm.next = p -> tm.next;
695 if (p != &first_thread) {
701 GC_API void GC_CALL GC_allow_register_threads(void)
703 /* Check GC is initialized and the current thread is registered. */
704 GC_ASSERT(GC_lookup_thread_inner(GetCurrentThreadId()) != 0);
706 # if !defined(GC_NO_THREADS_DISCOVERY) && !defined(PARALLEL_MARK)
707 /* GC_init() doesn't call GC_init_parallel() in this case. */
708 parallel_initialized = TRUE;
710 GC_need_to_lock = TRUE; /* We are multi-threaded now. */
713 GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *sb)
716 DWORD thread_id = GetCurrentThreadId();
719 if (GC_need_to_lock == FALSE)
720 ABORT("Threads explicit registering is not previously enabled");
722 /* We lock here, since we want to wait for an ongoing GC. */
724 me = GC_lookup_thread_inner(thread_id);
727 me = GC_register_my_thread_inner(sb, thread_id);
728 me -> flags |= DETACHED;
729 /* Treat as detached, since we do not need to worry about */
730 /* pointer results. */
732 GC_register_my_thread_inner(sb, thread_id);
738 /* else */ if ((me -> flags & FINISHED) != 0) {
739 GC_record_stack_base(me, sb);
740 me -> flags &= ~FINISHED; /* but not DETACHED */
741 # ifdef THREAD_LOCAL_ALLOC
742 GC_init_thread_local((GC_tlfs)(&me->tlfs));
754 /* Similar to that in pthread_support.c. */
755 STATIC void GC_wait_for_gc_completion(GC_bool wait_for_all)
757 GC_ASSERT(I_HOLD_LOCK());
758 if (GC_incremental && GC_collection_in_progress()) {
759 word old_gc_no = GC_gc_no;
761 /* Make sure that no part of our stack is still on the mark stack, */
762 /* since it's about to be unmapped. */
765 GC_in_thread_creation = TRUE;
766 GC_collect_a_little_inner(1);
767 GC_in_thread_creation = FALSE;
771 Sleep(0); /* yield */
773 } while (GC_incremental && GC_collection_in_progress()
774 && (wait_for_all || old_gc_no == GC_gc_no));
778 GC_API int GC_CALL GC_unregister_my_thread(void)
782 # ifdef DEBUG_THREADS
783 GC_log_printf("Unregistering thread 0x%lx\n", (long)GetCurrentThreadId());
786 if (GC_win32_dll_threads) {
787 # if defined(THREAD_LOCAL_ALLOC)
788 /* Can't happen: see GC_use_threads_discovery(). */
792 /* FIXME: If not DETACHED then just set FINISHED. */
794 /* FIXME: Should we just ignore this? */
795 GC_delete_thread(GetCurrentThreadId());
798 # if defined(THREAD_LOCAL_ALLOC) || defined(GC_PTHREADS)
801 DWORD thread_id = GetCurrentThreadId();
804 GC_wait_for_gc_completion(FALSE);
805 # if defined(THREAD_LOCAL_ALLOC) || defined(GC_PTHREADS)
806 me = GC_lookup_thread_inner(thread_id);
807 CHECK_LOOKUP_MY_THREAD(me);
808 GC_ASSERT(!KNOWN_FINISHED(me));
810 # if defined(THREAD_LOCAL_ALLOC)
811 GC_destroy_thread_local(&(me->tlfs));
814 if ((me -> flags & DETACHED) == 0) {
815 me -> flags |= FINISHED;
819 GC_delete_thread(thread_id);
826 /* Wrapper for functions that are likely to block for an appreciable */
827 /* length of time. */
829 /* GC_do_blocking_inner() is nearly the same as in pthread_support.c */
831 GC_INNER void GC_do_blocking_inner(ptr_t data, void * context)
833 struct blocking_data * d = (struct blocking_data *) data;
834 DWORD thread_id = GetCurrentThreadId();
837 ptr_t stack_ptr = GC_save_regs_in_stack();
842 me = GC_lookup_thread_inner(thread_id);
843 CHECK_LOOKUP_MY_THREAD(me);
844 GC_ASSERT(me -> thread_blocked_sp == NULL);
846 me -> backing_store_ptr = stack_ptr;
848 me -> thread_blocked_sp = (ptr_t) &d; /* save approx. sp */
849 /* Save context here if we want to support precise stack marking */
851 d -> client_data = (d -> fn)(d -> client_data);
852 LOCK(); /* This will block if the world is stopped. */
853 me -> thread_blocked_sp = NULL;
857 /* GC_call_with_gc_active() has the opposite to GC_do_blocking() */
858 /* functionality. It might be called from a user function invoked by */
859 /* GC_do_blocking() to temporarily back allow calling any GC function */
860 /* and/or manipulating pointers to the garbage collected heap. */
861 GC_API void * GC_CALL GC_call_with_gc_active(GC_fn_type fn,
864 struct GC_traced_stack_sect_s stacksect;
868 LOCK(); /* This will block if the world is stopped. */
869 me = GC_lookup_thread_inner(GetCurrentThreadId());
870 CHECK_LOOKUP_MY_THREAD(me);
871 /* Adjust our stack base value (this could happen unless */
872 /* GC_get_stack_base() was used which returned GC_SUCCESS). */
873 GC_ASSERT(me -> stack_base != NULL);
874 if (me -> stack_base < (ptr_t)(&stacksect))
875 me -> stack_base = (ptr_t)(&stacksect);
877 if (me -> thread_blocked_sp == NULL) {
878 /* We are not inside GC_do_blocking() - do nothing more. */
880 return fn(client_data);
883 /* Setup new "stack section". */
884 stacksect.saved_stack_ptr = me -> thread_blocked_sp;
886 /* This is the same as in GC_call_with_stack_base(). */
887 stacksect.backing_store_end = GC_save_regs_in_stack();
888 /* Unnecessarily flushes register stack, */
889 /* but that probably doesn't hurt. */
890 stacksect.saved_backing_store_ptr = me -> backing_store_ptr;
892 stacksect.prev = me -> traced_stack_sect;
893 me -> thread_blocked_sp = NULL;
894 me -> traced_stack_sect = &stacksect;
897 client_data = fn(client_data);
898 GC_ASSERT(me -> thread_blocked_sp == NULL);
899 GC_ASSERT(me -> traced_stack_sect == &stacksect);
901 /* Restore original "stack section". */
903 me -> traced_stack_sect = stacksect.prev;
905 me -> backing_store_ptr = stacksect.saved_backing_store_ptr;
907 me -> thread_blocked_sp = stacksect.saved_stack_ptr;
910 return client_data; /* result */
915 /* A quick-and-dirty cache of the mapping between pthread_t */
916 /* and win32 thread id. */
917 # define PTHREAD_MAP_SIZE 512
918 DWORD GC_pthread_map_cache[PTHREAD_MAP_SIZE] = {0};
919 # define PTHREAD_MAP_INDEX(pthread_id) \
920 ((NUMERIC_THREAD_ID(pthread_id) >> 5) % PTHREAD_MAP_SIZE)
921 /* It appears pthread_t is really a pointer type ... */
922 # define SET_PTHREAD_MAP_CACHE(pthread_id, win32_id) \
923 (GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)] = (win32_id))
924 # define GET_PTHREAD_MAP_CACHE(pthread_id) \
925 GC_pthread_map_cache[PTHREAD_MAP_INDEX(pthread_id)]
927 /* Return a GC_thread corresponding to a given pthread_t. */
928 /* Returns 0 if it's not there. */
929 /* We assume that this is only called for pthread ids that */
930 /* have not yet terminated or are still joinable, and */
931 /* cannot be concurrently terminated. */
932 /* Assumes we do NOT hold the allocation lock. */
933 STATIC GC_thread GC_lookup_pthread(pthread_t id)
935 # ifndef GC_NO_THREADS_DISCOVERY
936 if (GC_win32_dll_threads) {
938 LONG my_max = GC_get_max_thread_index();
940 for (i = 0; i <= my_max &&
941 (!AO_load_acquire(&dll_thread_table[i].tm.in_use)
942 || THREAD_EQUAL(dll_thread_table[i].pthread_id, id));
943 /* Must still be in_use, since nobody else can */
944 /* store our thread_id. */
948 return i <= my_max ? (GC_thread)(dll_thread_table + i) : NULL;
952 /* We first try the cache. If that fails, we use a very slow */
954 word hv_guess = THREAD_TABLE_INDEX(GET_PTHREAD_MAP_CACHE(id));
960 for (p = GC_threads[hv_guess]; 0 != p; p = p -> tm.next) {
961 if (THREAD_EQUAL(p -> pthread_id, id))
964 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
965 for (p = GC_threads[hv]; 0 != p; p = p -> tm.next) {
966 if (THREAD_EQUAL(p -> pthread_id, id))
977 #endif /* GC_PTHREADS */
979 #ifdef CAN_HANDLE_FORK
980 /* Similar to that in pthread_support.c but also rehashes the table */
981 /* since hash map key (thread_id) differs from that in the parent. */
982 STATIC void GC_remove_all_threads_but_me(void)
985 GC_thread p, next, me = NULL;
987 pthread_t pthread_id = pthread_self(); /* same as in parent */
989 GC_ASSERT(!GC_win32_dll_threads);
990 for (hv = 0; hv < THREAD_TABLE_SZ; ++hv) {
991 for (p = GC_threads[hv]; 0 != p; p = next) {
993 if (THREAD_EQUAL(p -> pthread_id, pthread_id)) {
994 GC_ASSERT(me == NULL);
998 # ifdef THREAD_LOCAL_ALLOC
999 if ((p -> flags & FINISHED) == 0) {
1000 GC_destroy_thread_local(&p->tlfs);
1003 if (&first_thread != p)
1004 GC_INTERNAL_FREE(p);
1007 GC_threads[hv] = NULL;
1010 /* Put "me" back to GC_threads. */
1011 GC_ASSERT(me != NULL);
1012 thread_id = GetCurrentThreadId(); /* differs from that in parent */
1013 GC_threads[THREAD_TABLE_INDEX(thread_id)] = me;
1015 /* Update Win32 thread Id and handle. */
1016 me -> id = thread_id;
1018 if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1019 GetCurrentProcess(), (HANDLE *)&me->handle,
1020 0 /* dwDesiredAccess */, FALSE /* bInheritHandle */,
1021 DUPLICATE_SAME_ACCESS))
1022 ABORT("DuplicateHandle failed");
1025 # if defined(THREAD_LOCAL_ALLOC) && !defined(USE_CUSTOM_SPECIFIC)
1026 /* For Cygwin, we need to re-assign thread-local pointer to */
1027 /* 'tlfs' (it is ok to call GC_destroy_thread_local and */
1028 /* GC_free_internal before this action). */
1029 if (GC_setspecific(GC_thread_key, &me->tlfs) != 0)
1030 ABORT("GC_setspecific failed (in child)");
1034 STATIC void GC_fork_prepare_proc(void)
1037 # ifdef PARALLEL_MARK
1039 GC_wait_for_reclaim();
1041 GC_wait_for_gc_completion(TRUE);
1042 # ifdef PARALLEL_MARK
1044 GC_acquire_mark_lock();
1048 STATIC void GC_fork_parent_proc(void)
1050 # ifdef PARALLEL_MARK
1052 GC_release_mark_lock();
1057 STATIC void GC_fork_child_proc(void)
1059 # ifdef PARALLEL_MARK
1061 GC_release_mark_lock();
1063 GC_parallel = FALSE;
1064 /* Turn off parallel marking in the child, since we are */
1065 /* probably just going to exec, and we would have to */
1066 /* restart mark threads. */
1069 GC_remove_all_threads_but_me();
1072 #endif /* CAN_HANDLE_FORK */
1074 void GC_push_thread_structures(void)
1076 GC_ASSERT(I_HOLD_LOCK());
1077 # ifndef GC_NO_THREADS_DISCOVERY
1078 if (GC_win32_dll_threads) {
1079 /* Unlike the other threads implementations, the thread table here */
1080 /* contains no pointers to the collectable heap. Thus we have */
1081 /* no private structures we need to preserve. */
1083 int i; /* pthreads may keep a pointer in the thread exit value */
1084 LONG my_max = GC_get_max_thread_index();
1086 for (i = 0; i <= my_max; i++)
1087 if (dll_thread_table[i].tm.in_use)
1088 GC_push_all((ptr_t)&(dll_thread_table[i].status),
1089 (ptr_t)(&(dll_thread_table[i].status)+1));
1094 GC_push_all((ptr_t)(GC_threads), (ptr_t)(GC_threads)+sizeof(GC_threads));
1096 # if defined(THREAD_LOCAL_ALLOC)
1097 GC_push_all((ptr_t)(&GC_thread_key),
1098 (ptr_t)(&GC_thread_key)+sizeof(&GC_thread_key));
1099 /* Just in case we ever use our own TLS implementation. */
1103 /* Suspend the given thread, if it's still active. */
1104 STATIC void GC_suspend(GC_thread t)
1107 /* Apparently the Windows 95 GetOpenFileName call creates */
1108 /* a thread that does not properly get cleaned up, and */
1109 /* SuspendThread on its descriptor may provoke a crash. */
1110 /* This reduces the probability of that event, though it still */
1111 /* appears there's a race here. */
1114 UNPROTECT_THREAD(t);
1116 if (GetExitCodeThread(t -> handle, &exitCode) &&
1117 exitCode != STILL_ACTIVE) {
1119 t -> stack_base = 0; /* prevent stack from being pushed */
1121 /* this breaks pthread_join on Cygwin, which is guaranteed to */
1122 /* only see user pthreads */
1123 GC_ASSERT(GC_win32_dll_threads);
1124 GC_delete_gc_thread(t);
1129 # if defined(MPROTECT_VDB)
1130 /* Acquire the spin lock we use to update dirty bits. */
1131 /* Threads shouldn't get stopped holding it. But we may */
1132 /* acquire and release it in the UNPROTECT_THREAD call. */
1133 while (AO_test_and_set_acquire(&GC_fault_handler_lock) == AO_TS_SET) {
1139 /* SuspendThread() will fail if thread is running kernel code. */
1140 while (SuspendThread(THREAD_HANDLE(t)) == (DWORD)-1)
1141 Sleep(10); /* in millis */
1143 if (SuspendThread(t -> handle) == (DWORD)-1)
1144 ABORT("SuspendThread failed");
1145 # endif /* !MSWINCE */
1146 t -> suspended = (unsigned char)TRUE;
1147 # if defined(MPROTECT_VDB)
1148 AO_CLEAR(&GC_fault_handler_lock);
1152 #if defined(GC_ASSERTIONS) && !defined(CYGWIN32)
1153 GC_INNER GC_bool GC_write_disabled = FALSE;
1154 /* TRUE only if GC_stop_world() acquired GC_write_cs. */
1157 GC_INNER void GC_stop_world(void)
1159 DWORD thread_id = GetCurrentThreadId();
1161 if (!GC_thr_initialized)
1162 ABORT("GC_stop_world() called before GC_thr_init()");
1163 GC_ASSERT(I_HOLD_LOCK());
1165 /* This code is the same as in pthread_stop_world.c */
1166 # ifdef PARALLEL_MARK
1168 GC_acquire_mark_lock();
1169 GC_ASSERT(GC_fl_builder_count == 0);
1170 /* We should have previously waited for it to become zero. */
1172 # endif /* PARALLEL_MARK */
1174 # if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
1175 GC_please_stop = TRUE;
1178 GC_ASSERT(!GC_write_disabled);
1179 EnterCriticalSection(&GC_write_cs);
1180 /* It's not allowed to call GC_printf() (and friends) here down to */
1181 /* LeaveCriticalSection (same applies recursively to */
1182 /* GC_get_max_thread_index(), GC_suspend(), GC_delete_gc_thread() */
1183 /* (only if GC_win32_dll_threads), GC_size() and */
1184 /* GC_remove_protection()). */
1185 # ifdef GC_ASSERTIONS
1186 GC_write_disabled = TRUE;
1189 # ifndef GC_NO_THREADS_DISCOVERY
1190 if (GC_win32_dll_threads) {
1193 /* Any threads being created during this loop will end up setting */
1194 /* GC_attached_thread when they start. This will force marking */
1195 /* to restart. This is not ideal, but hopefully correct. */
1196 GC_attached_thread = FALSE;
1197 my_max = (int)GC_get_max_thread_index();
1198 for (i = 0; i <= my_max; i++) {
1199 GC_vthread t = dll_thread_table + i;
1200 if (t -> stack_base != 0 && t -> thread_blocked_sp == NULL
1201 && t -> id != thread_id) {
1202 GC_suspend((GC_thread)t);
1211 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1212 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1213 if (t -> stack_base != 0 && t -> thread_blocked_sp == NULL
1214 && !KNOWN_FINISHED(t) && t -> id != thread_id) {
1221 # ifdef GC_ASSERTIONS
1222 GC_write_disabled = FALSE;
1224 LeaveCriticalSection(&GC_write_cs);
1226 # ifdef PARALLEL_MARK
1228 GC_release_mark_lock();
1232 GC_INNER void GC_start_world(void)
1234 # ifdef GC_ASSERTIONS
1235 DWORD thread_id = GetCurrentThreadId();
1239 GC_ASSERT(I_HOLD_LOCK());
1240 if (GC_win32_dll_threads) {
1241 LONG my_max = GC_get_max_thread_index();
1242 for (i = 0; i <= my_max; i++) {
1243 GC_thread t = (GC_thread)(dll_thread_table + i);
1244 if (t -> suspended) {
1245 GC_ASSERT(t -> stack_base != 0 && t -> id != thread_id);
1246 if (ResumeThread(THREAD_HANDLE(t)) == (DWORD)-1)
1247 ABORT("ResumeThread failed");
1248 t -> suspended = FALSE;
1255 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1256 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1257 if (t -> suspended) {
1258 GC_ASSERT(t -> stack_base != 0 && t -> id != thread_id);
1259 if (ResumeThread(THREAD_HANDLE(t)) == (DWORD)-1)
1260 ABORT("ResumeThread failed");
1261 UNPROTECT_THREAD(t);
1262 t -> suspended = FALSE;
1267 # if !defined(GC_NO_THREADS_DISCOVERY) || defined(GC_ASSERTIONS)
1268 GC_please_stop = FALSE;
1273 /* The VirtualQuery calls below won't work properly on some old WinCE */
1274 /* versions, but since each stack is restricted to an aligned 64 KiB */
1275 /* region of virtual memory we can just take the next lowest multiple */
1276 /* of 64 KiB. The result of this macro must not be used as its */
1277 /* argument later and must not be used as the lower bound for sp */
1278 /* check (since the stack may be bigger than 64 KiB). */
1279 # define GC_wince_evaluate_stack_min(s) \
1280 (ptr_t)(((word)(s) - 1) & ~(word)0xFFFF)
1281 #elif defined(GC_ASSERTIONS)
1282 # define GC_dont_query_stack_min FALSE
1285 /* A cache holding the results of the recent VirtualQuery call. */
1286 /* Protected by the allocation lock. */
1287 static ptr_t last_address = 0;
1288 static MEMORY_BASIC_INFORMATION last_info;
1290 /* Probe stack memory region (starting at "s") to find out its */
1291 /* lowest address (i.e. stack top). */
1292 /* S must be a mapped address inside the region, NOT the first */
1293 /* unmapped address. */
1294 STATIC ptr_t GC_get_stack_min(ptr_t s)
1298 GC_ASSERT(I_HOLD_LOCK());
1299 if (s != last_address) {
1300 VirtualQuery(s, &last_info, sizeof(last_info));
1304 bottom = last_info.BaseAddress;
1305 VirtualQuery(bottom - 1, &last_info, sizeof(last_info));
1306 last_address = bottom - 1;
1307 } while ((last_info.Protect & PAGE_READWRITE)
1308 && !(last_info.Protect & PAGE_GUARD));
1312 /* Return true if the page at s has protections appropriate */
1313 /* for a stack page. */
1314 static GC_bool may_be_in_stack(ptr_t s)
1316 GC_ASSERT(I_HOLD_LOCK());
1317 if (s != last_address) {
1318 VirtualQuery(s, &last_info, sizeof(last_info));
1321 return (last_info.Protect & PAGE_READWRITE)
1322 && !(last_info.Protect & PAGE_GUARD);
1325 STATIC word GC_push_stack_for(GC_thread thread, DWORD me)
1327 ptr_t sp, stack_min;
1329 struct GC_traced_stack_sect_s *traced_stack_sect =
1330 thread -> traced_stack_sect;
1331 if (thread -> id == me) {
1332 GC_ASSERT(thread -> thread_blocked_sp == NULL);
1333 sp = GC_approx_sp();
1334 } else if ((sp = thread -> thread_blocked_sp) == NULL) {
1335 /* Use saved sp value for blocked threads. */
1336 /* For unblocked threads call GetThreadContext(). */
1338 context.ContextFlags = CONTEXT_INTEGER|CONTEXT_CONTROL;
1339 if (!GetThreadContext(THREAD_HANDLE(thread), &context))
1340 ABORT("GetThreadContext failed");
1342 /* Push all registers that might point into the heap. Frame */
1343 /* pointer registers are included in case client code was */
1344 /* compiled with the 'omit frame pointer' optimisation. */
1345 # define PUSH1(reg) GC_push_one((word)context.reg)
1346 # define PUSH2(r1,r2) PUSH1(r1), PUSH1(r2)
1347 # define PUSH4(r1,r2,r3,r4) PUSH2(r1,r2), PUSH2(r3,r4)
1349 PUSH4(Edi,Esi,Ebx,Edx), PUSH2(Ecx,Eax), PUSH1(Ebp);
1350 sp = (ptr_t)context.Esp;
1351 # elif defined(X86_64)
1352 PUSH4(Rax,Rcx,Rdx,Rbx); PUSH2(Rbp, Rsi); PUSH1(Rdi);
1353 PUSH4(R8, R9, R10, R11); PUSH4(R12, R13, R14, R15);
1354 sp = (ptr_t)context.Rsp;
1355 # elif defined(ARM32)
1356 PUSH4(R0,R1,R2,R3),PUSH4(R4,R5,R6,R7),PUSH4(R8,R9,R10,R11);
1358 sp = (ptr_t)context.Sp;
1360 PUSH4(R0,R1,R2,R3), PUSH4(R4,R5,R6,R7), PUSH4(R8,R9,R10,R11);
1361 PUSH2(R12,R13), PUSH1(R14);
1362 sp = (ptr_t)context.R15;
1363 # elif defined(MIPS)
1364 PUSH4(IntAt,IntV0,IntV1,IntA0), PUSH4(IntA1,IntA2,IntA3,IntT0);
1365 PUSH4(IntT1,IntT2,IntT3,IntT4), PUSH4(IntT5,IntT6,IntT7,IntS0);
1366 PUSH4(IntS1,IntS2,IntS3,IntS4), PUSH4(IntS5,IntS6,IntS7,IntT8);
1367 PUSH4(IntT9,IntK0,IntK1,IntS8);
1368 sp = (ptr_t)context.IntSp;
1370 PUSH4(Gpr0, Gpr3, Gpr4, Gpr5), PUSH4(Gpr6, Gpr7, Gpr8, Gpr9);
1371 PUSH4(Gpr10,Gpr11,Gpr12,Gpr14), PUSH4(Gpr15,Gpr16,Gpr17,Gpr18);
1372 PUSH4(Gpr19,Gpr20,Gpr21,Gpr22), PUSH4(Gpr23,Gpr24,Gpr25,Gpr26);
1373 PUSH4(Gpr27,Gpr28,Gpr29,Gpr30), PUSH1(Gpr31);
1374 sp = (ptr_t)context.Gpr1;
1375 # elif defined(ALPHA)
1376 PUSH4(IntV0,IntT0,IntT1,IntT2), PUSH4(IntT3,IntT4,IntT5,IntT6);
1377 PUSH4(IntT7,IntS0,IntS1,IntS2), PUSH4(IntS3,IntS4,IntS5,IntFp);
1378 PUSH4(IntA0,IntA1,IntA2,IntA3), PUSH4(IntA4,IntA5,IntT8,IntT9);
1379 PUSH4(IntT10,IntT11,IntT12,IntAt);
1380 sp = (ptr_t)context.IntSp;
1382 # error "architecture is not supported"
1384 } /* ! current thread */
1386 /* Set stack_min to the lowest address in the thread stack, */
1387 /* or to an address in the thread stack no larger than sp, */
1388 /* taking advantage of the old value to avoid slow traversals */
1389 /* of large stacks. */
1390 if (thread -> last_stack_min == ADDR_LIMIT) {
1392 if (GC_dont_query_stack_min) {
1393 stack_min = GC_wince_evaluate_stack_min(traced_stack_sect != NULL ?
1394 (ptr_t)traced_stack_sect : thread -> stack_base);
1395 /* Keep last_stack_min value unmodified. */
1399 stack_min = GC_get_stack_min(traced_stack_sect != NULL ?
1400 (ptr_t)traced_stack_sect : thread -> stack_base);
1401 UNPROTECT_THREAD(thread);
1402 thread -> last_stack_min = stack_min;
1405 /* First, adjust the latest known minimum stack address if we */
1406 /* are inside GC_call_with_gc_active(). */
1407 if (traced_stack_sect != NULL &&
1408 thread -> last_stack_min > (ptr_t)traced_stack_sect) {
1409 UNPROTECT_THREAD(thread);
1410 thread -> last_stack_min = (ptr_t)traced_stack_sect;
1413 if (sp < thread -> stack_base && sp >= thread -> last_stack_min) {
1416 /* In the current thread it is always safe to use sp value. */
1417 if (may_be_in_stack(thread -> id == me &&
1418 sp < thread -> last_stack_min ?
1419 sp : thread -> last_stack_min)) {
1420 stack_min = last_info.BaseAddress;
1421 /* Do not probe rest of the stack if sp is correct. */
1422 if (sp < stack_min || sp >= thread->stack_base)
1423 stack_min = GC_get_stack_min(thread -> last_stack_min);
1425 /* Stack shrunk? Is this possible? */
1426 stack_min = GC_get_stack_min(thread -> stack_base);
1428 UNPROTECT_THREAD(thread);
1429 thread -> last_stack_min = stack_min;
1433 GC_ASSERT(GC_dont_query_stack_min
1434 || stack_min == GC_get_stack_min(thread -> stack_base)
1435 || (sp >= stack_min && stack_min < thread -> stack_base
1436 && stack_min > GC_get_stack_min(thread -> stack_base)));
1438 if (sp >= stack_min && sp < thread->stack_base) {
1439 # ifdef DEBUG_THREADS
1440 GC_log_printf("Pushing stack for 0x%x from sp %p to %p from 0x%x\n",
1441 (int)thread -> id, sp, thread -> stack_base, (int)me);
1443 GC_push_all_stack_sections(sp, thread->stack_base, traced_stack_sect);
1445 /* If not current thread then it is possible for sp to point to */
1446 /* the guarded (untouched yet) page just below the current */
1447 /* stack_min of the thread. */
1448 if (thread -> id == me || sp >= thread->stack_base
1449 || sp + GC_page_size < stack_min)
1450 WARN("Thread stack pointer %p out of range, pushing everything\n",
1452 # ifdef DEBUG_THREADS
1453 GC_log_printf("Pushing stack for 0x%x from (min) %p to %p from 0x%x\n",
1454 (int)thread->id, stack_min, thread->stack_base, (int)me);
1456 /* Push everything - ignore "traced stack section" data. */
1457 GC_push_all_stack(stack_min, thread->stack_base);
1459 return thread->stack_base - sp; /* stack grows down */
1462 GC_INNER void GC_push_all_stacks(void)
1464 DWORD thread_id = GetCurrentThreadId();
1465 GC_bool found_me = FALSE;
1466 # ifndef SMALL_CONFIG
1467 unsigned nthreads = 0;
1469 word total_size = 0;
1470 # ifndef GC_NO_THREADS_DISCOVERY
1471 if (GC_win32_dll_threads) {
1473 LONG my_max = GC_get_max_thread_index();
1475 for (i = 0; i <= my_max; i++) {
1476 GC_thread t = (GC_thread)(dll_thread_table + i);
1477 if (t -> tm.in_use && t -> stack_base) {
1478 # ifndef SMALL_CONFIG
1481 total_size += GC_push_stack_for(t, thread_id);
1482 if (t -> id == thread_id) found_me = TRUE;
1489 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1491 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1492 if (!KNOWN_FINISHED(t) && t -> stack_base) {
1493 # ifndef SMALL_CONFIG
1496 total_size += GC_push_stack_for(t, thread_id);
1497 if (t -> id == thread_id) found_me = TRUE;
1502 # ifndef SMALL_CONFIG
1503 if (GC_print_stats == VERBOSE) {
1504 GC_log_printf("Pushed %d thread stacks%s\n", nthreads,
1505 GC_win32_dll_threads ? " based on DllMain thread tracking" : "");
1508 if (!found_me && !GC_in_thread_creation)
1509 ABORT("Collecting from unknown thread");
1510 GC_total_stacksize = total_size;
1513 #ifdef PARALLEL_MARK
1515 # ifndef MAX_MARKERS
1516 # define MAX_MARKERS 16
1519 static ptr_t marker_sp[MAX_MARKERS - 1]; /* The cold end of the stack */
1522 static ptr_t marker_bsp[MAX_MARKERS - 1];
1525 static ptr_t marker_last_stack_min[MAX_MARKERS - 1];
1526 /* Last known minimum (hottest) address */
1527 /* in stack (or ADDR_LIMIT if unset) */
1530 #endif /* PARALLEL_MARK */
1532 /* Find stack with the lowest address which overlaps the */
1533 /* interval [start, limit). */
1534 /* Return stack bounds in *lo and *hi. If no such stack */
1535 /* is found, both *hi and *lo will be set to an address */
1536 /* higher than limit. */
1537 GC_INNER void GC_get_next_stack(char *start, char *limit,
1538 char **lo, char **hi)
1541 char * current_min = ADDR_LIMIT; /* Least in-range stack base */
1542 ptr_t *plast_stack_min = NULL; /* Address of last_stack_min */
1543 /* field for thread corresponding */
1544 /* to current_min. */
1545 GC_thread thread = NULL; /* Either NULL or points to the */
1546 /* thread's hash table entry */
1547 /* containing *plast_stack_min. */
1549 /* First set current_min, ignoring limit. */
1550 if (GC_win32_dll_threads) {
1551 LONG my_max = GC_get_max_thread_index();
1553 for (i = 0; i <= my_max; i++) {
1554 ptr_t s = (ptr_t)(dll_thread_table[i].stack_base);
1556 if (s > start && s < current_min) {
1557 /* Update address of last_stack_min. */
1558 plast_stack_min = (ptr_t * /* no volatile */)
1559 &dll_thread_table[i].last_stack_min;
1564 for (i = 0; i < THREAD_TABLE_SZ; i++) {
1567 for (t = GC_threads[i]; t != 0; t = t -> tm.next) {
1568 ptr_t s = t -> stack_base;
1570 if (s > start && s < current_min) {
1571 /* Update address of last_stack_min. */
1572 plast_stack_min = &t -> last_stack_min;
1573 thread = t; /* Remember current thread to unprotect. */
1578 # ifdef PARALLEL_MARK
1579 for (i = 0; i < GC_markers - 1; ++i) {
1580 ptr_t s = marker_sp[i];
1582 /* FIXME: not implemented */
1584 if (s > start && s < current_min) {
1585 GC_ASSERT(marker_last_stack_min[i] != NULL);
1586 plast_stack_min = &marker_last_stack_min[i];
1588 thread = NULL; /* Not a thread's hash table entry. */
1595 if (current_min == ADDR_LIMIT) {
1600 GC_ASSERT(current_min > start && plast_stack_min != NULL);
1602 if (GC_dont_query_stack_min) {
1603 *lo = GC_wince_evaluate_stack_min(current_min);
1604 /* Keep last_stack_min value unmodified. */
1609 if (current_min > limit && !may_be_in_stack(limit)) {
1610 /* Skip the rest since the memory region at limit address is */
1611 /* not a stack (so the lowest address of the found stack would */
1612 /* be above the limit value anyway). */
1617 /* Get the minimum address of the found stack by probing its memory */
1618 /* region starting from the recent known minimum (if set). */
1619 if (*plast_stack_min == ADDR_LIMIT
1620 || !may_be_in_stack(*plast_stack_min)) {
1621 /* Unsafe to start from last_stack_min value. */
1622 *lo = GC_get_stack_min(current_min);
1624 /* Use the recent value to optimize search for min address. */
1625 *lo = GC_get_stack_min(*plast_stack_min);
1628 /* Remember current stack_min value. */
1629 if (thread != NULL) {
1630 UNPROTECT_THREAD(thread);
1632 *plast_stack_min = *lo;
1635 #ifdef PARALLEL_MARK
1637 # if defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK)
1638 /* Use pthread-based parallel mark implementation. */
1639 # define GC_PTHREADS_PARAMARK
1642 /* GC_mark_thread() is the same as in pthread_support.c */
1643 # ifdef GC_PTHREADS_PARAMARK
1644 STATIC void * GC_mark_thread(void * id)
1647 STATIC DWORD WINAPI GC_mark_thread(LPVOID id)
1649 STATIC unsigned __stdcall GC_mark_thread(void * id)
1653 word my_mark_no = 0;
1655 if ((word)id == (word)-1) return 0; /* to make compiler happy */
1656 marker_sp[(word)id] = GC_approx_sp();
1658 marker_bsp[(word)id] = GC_save_regs_in_stack();
1661 for (;; ++my_mark_no) {
1662 if (my_mark_no - GC_mark_no > (word)2) {
1663 /* resynchronize if we get far off, e.g. because GC_mark_no */
1665 my_mark_no = GC_mark_no;
1667 # ifdef DEBUG_THREADS
1668 GC_log_printf("Starting mark helper for mark number %lu\n",
1669 (unsigned long)my_mark_no);
1671 GC_help_marker(my_mark_no);
1675 # ifdef GC_ASSERTIONS
1676 GC_INNER unsigned long GC_mark_lock_holder = NO_THREAD;
1679 /* GC_mark_threads[] is unused here unlike that in pthread_support.c */
1681 # ifdef GC_PTHREADS_PARAMARK
1682 # include <pthread.h>
1684 # ifndef NUMERIC_THREAD_ID
1685 # define NUMERIC_THREAD_ID(id) (unsigned long)GC_PTHREAD_PTRVAL(id)
1688 /* start_mark_threads() is the same as in pthread_support.c except for: */
1689 /* - GC_markers value is adjusted already; */
1690 /* - thread stack is assumed to be large enough; and */
1691 /* - statistics about the number of marker threads is printed outside. */
1692 static void start_mark_threads(void)
1695 pthread_attr_t attr;
1696 pthread_t new_thread;
1698 if (0 != pthread_attr_init(&attr)) ABORT("pthread_attr_init failed");
1700 if (0 != pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED))
1701 ABORT("pthread_attr_setdetachstate failed");
1703 for (i = 0; i < GC_markers - 1; ++i) {
1704 marker_last_stack_min[i] = ADDR_LIMIT;
1705 if (0 != pthread_create(&new_thread, &attr,
1706 GC_mark_thread, (void *)(word)i)) {
1707 WARN("Marker thread creation failed.\n", 0);
1708 /* Don't try to create other marker threads. */
1710 if (i == 0) GC_parallel = FALSE;
1714 pthread_attr_destroy(&attr);
1717 static pthread_mutex_t mark_mutex = PTHREAD_MUTEX_INITIALIZER;
1719 static pthread_cond_t builder_cv = PTHREAD_COND_INITIALIZER;
1721 /* GC_acquire/release_mark_lock(), GC_wait_builder/marker(), */
1722 /* GC_wait_for_reclaim(), GC_notify_all_builder/marker() are the same */
1723 /* as in pthread_support.c except that GC_generic_lock() is not used. */
1726 AO_t GC_block_count = 0;
1729 GC_INNER void GC_acquire_mark_lock(void)
1731 if (pthread_mutex_lock(&mark_mutex) != 0) {
1732 ABORT("pthread_mutex_lock failed");
1735 (void)AO_fetch_and_add1(&GC_block_count);
1737 /* GC_generic_lock(&mark_mutex); */
1738 # ifdef GC_ASSERTIONS
1739 GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
1743 GC_INNER void GC_release_mark_lock(void)
1745 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
1746 # ifdef GC_ASSERTIONS
1747 GC_mark_lock_holder = NO_THREAD;
1749 if (pthread_mutex_unlock(&mark_mutex) != 0) {
1750 ABORT("pthread_mutex_unlock failed");
1754 /* Collector must wait for a freelist builders for 2 reasons: */
1755 /* 1) Mark bits may still be getting examined without lock. */
1756 /* 2) Partial free lists referenced only by locals may not be */
1757 /* scanned correctly, e.g. if they contain "pointer-free" objects, */
1758 /* since the free-list link may be ignored. */
1759 STATIC void GC_wait_builder(void)
1761 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
1762 # ifdef GC_ASSERTIONS
1763 GC_mark_lock_holder = NO_THREAD;
1765 if (pthread_cond_wait(&builder_cv, &mark_mutex) != 0) {
1766 ABORT("pthread_cond_wait failed");
1768 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
1769 # ifdef GC_ASSERTIONS
1770 GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
1774 GC_INNER void GC_wait_for_reclaim(void)
1776 GC_acquire_mark_lock();
1777 while (GC_fl_builder_count > 0) {
1780 GC_release_mark_lock();
1783 GC_INNER void GC_notify_all_builder(void)
1785 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
1786 if (pthread_cond_broadcast(&builder_cv) != 0) {
1787 ABORT("pthread_cond_broadcast failed");
1791 static pthread_cond_t mark_cv = PTHREAD_COND_INITIALIZER;
1793 GC_INNER void GC_wait_marker(void)
1795 GC_ASSERT(GC_mark_lock_holder == NUMERIC_THREAD_ID(pthread_self()));
1796 # ifdef GC_ASSERTIONS
1797 GC_mark_lock_holder = NO_THREAD;
1799 if (pthread_cond_wait(&mark_cv, &mark_mutex) != 0) {
1800 ABORT("pthread_cond_wait failed");
1802 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
1803 # ifdef GC_ASSERTIONS
1804 GC_mark_lock_holder = NUMERIC_THREAD_ID(pthread_self());
1808 GC_INNER void GC_notify_all_marker(void)
1810 if (pthread_cond_broadcast(&mark_cv) != 0) {
1811 ABORT("pthread_cond_broadcast failed");
1815 # else /* ! GC_PTHREADS_PARAMARK */
1817 # ifdef DONT_USE_SIGNALANDWAIT
1818 STATIC HANDLE GC_marker_cv[MAX_MARKERS - 1] = {0};
1819 /* Events with manual reset (one for each */
1822 STATIC DWORD GC_marker_Id[MAX_MARKERS - 1] = {0};
1823 /* This table is used for mapping helper */
1824 /* threads ID to mark helper index (linear */
1825 /* search is used since the mapping contains */
1826 /* only a few entries). */
1829 # ifndef MARK_THREAD_STACK_SIZE
1830 # define MARK_THREAD_STACK_SIZE 0 /* default value */
1833 /* mark_mutex_event, builder_cv, mark_cv are initialized in GC_thr_init */
1834 static HANDLE mark_mutex_event = (HANDLE)0; /* Event with auto-reset. */
1835 static HANDLE builder_cv = (HANDLE)0; /* Event with manual reset. */
1836 static HANDLE mark_cv = (HANDLE)0; /* Event with manual reset. */
1838 static void start_mark_threads(void)
1845 GC_uintptr_t handle;
1849 # ifdef DONT_USE_SIGNALANDWAIT
1850 /* Initialize GC_marker_cv[] and GC_marker_Id[] fully before */
1851 /* starting the first helper thread. */
1852 for (i = 0; i < GC_markers - 1; ++i) {
1853 GC_marker_Id[i] = GetCurrentThreadId();
1854 if ((GC_marker_cv[i] = CreateEvent(NULL /* attrs */,
1855 TRUE /* isManualReset */,
1856 FALSE /* initialState */,
1857 NULL /* name (A/W) */)) == (HANDLE)0)
1858 ABORT("CreateEvent() failed");
1862 for (i = 0; i < GC_markers - 1; ++i) {
1863 marker_last_stack_min[i] = ADDR_LIMIT;
1865 /* There is no _beginthreadex() in WinCE. */
1866 handle = CreateThread(NULL /* lpsa */,
1867 MARK_THREAD_STACK_SIZE /* ignored */,
1868 GC_mark_thread, (LPVOID)(word)i,
1869 0 /* fdwCreate */, &thread_id);
1870 if (handle == NULL) {
1871 WARN("Marker thread creation failed\n", 0);
1872 /* The most probable failure reason is "not enough memory". */
1873 /* Don't try to create other marker threads. */
1876 /* It's safe to detach the thread. */
1877 CloseHandle(handle);
1880 handle = _beginthreadex(NULL /* security_attr */,
1881 MARK_THREAD_STACK_SIZE, GC_mark_thread,
1882 (void *)(word)i, 0 /* flags */, &thread_id);
1883 if (!handle || handle == (GC_uintptr_t)-1L) {
1884 WARN("Marker thread creation failed\n", 0);
1885 /* Don't try to create other marker threads. */
1887 } else {/* We may detach the thread (if handle is of HANDLE type) */
1888 /* CloseHandle((HANDLE)handle); */
1893 /* Adjust GC_markers (and free unused resources) in case of failure. */
1894 # ifdef DONT_USE_SIGNALANDWAIT
1895 while ((int)GC_markers > i + 1) {
1897 CloseHandle(GC_marker_cv[(int)GC_markers - 1]);
1903 GC_parallel = FALSE;
1904 CloseHandle(mark_cv);
1905 CloseHandle(builder_cv);
1906 CloseHandle(mark_mutex_event);
1910 # ifdef DONT_USE_SIGNALANDWAIT
1911 STATIC /* volatile */ LONG GC_mark_mutex_state = 0;
1912 /* Mutex state: 0 - unlocked, */
1913 /* 1 - locked and no other waiters, */
1914 /* -1 - locked and waiters may exist. */
1915 /* Accessed by InterlockedExchange(). */
1917 STATIC volatile AO_t GC_mark_mutex_waitcnt = 0;
1918 /* Number of waiters + 1; 0 - unlocked. */
1921 /* #define LOCK_STATS */
1923 AO_t GC_block_count = 0;
1924 AO_t GC_unlocked_count = 0;
1927 GC_INNER void GC_acquire_mark_lock(void)
1929 # ifdef DONT_USE_SIGNALANDWAIT
1930 if (InterlockedExchange(&GC_mark_mutex_state, 1 /* locked */) != 0)
1932 if (AO_fetch_and_add1_acquire(&GC_mark_mutex_waitcnt) != 0)
1936 (void)AO_fetch_and_add1(&GC_block_count);
1938 # ifdef DONT_USE_SIGNALANDWAIT
1939 /* Repeatedly reset the state and wait until acquire the lock. */
1940 while (InterlockedExchange(&GC_mark_mutex_state,
1941 -1 /* locked_and_has_waiters */) != 0)
1944 if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
1945 ABORT("WaitForSingleObject() failed");
1950 (void)AO_fetch_and_add1(&GC_unlocked_count);
1954 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
1955 # ifdef GC_ASSERTIONS
1956 GC_mark_lock_holder = (unsigned long)GetCurrentThreadId();
1960 GC_INNER void GC_release_mark_lock(void)
1962 GC_ASSERT(GC_mark_lock_holder == (unsigned long)GetCurrentThreadId());
1963 # ifdef GC_ASSERTIONS
1964 GC_mark_lock_holder = NO_THREAD;
1966 # ifdef DONT_USE_SIGNALANDWAIT
1967 if (InterlockedExchange(&GC_mark_mutex_state, 0 /* unlocked */) < 0)
1969 GC_ASSERT(AO_load(&GC_mark_mutex_waitcnt) != 0);
1970 if (AO_fetch_and_sub1_release(&GC_mark_mutex_waitcnt) > 1)
1974 if (SetEvent(mark_mutex_event) == FALSE)
1975 ABORT("SetEvent() failed");
1979 /* In GC_wait_for_reclaim/GC_notify_all_builder() we emulate POSIX */
1980 /* cond_wait/cond_broadcast() primitives with WinAPI Event object */
1981 /* (working in "manual reset" mode). This works here because */
1982 /* GC_notify_all_builder() is always called holding lock on */
1983 /* mark_mutex and the checked condition (GC_fl_builder_count == 0) */
1984 /* is the only one for which broadcasting on builder_cv is performed. */
1986 GC_INNER void GC_wait_for_reclaim(void)
1988 GC_ASSERT(builder_cv != 0);
1990 GC_acquire_mark_lock();
1991 if (GC_fl_builder_count == 0)
1993 if (ResetEvent(builder_cv) == FALSE)
1994 ABORT("ResetEvent() failed");
1995 GC_release_mark_lock();
1996 if (WaitForSingleObject(builder_cv, INFINITE) == WAIT_FAILED)
1997 ABORT("WaitForSingleObject() failed");
1999 GC_release_mark_lock();
2002 GC_INNER void GC_notify_all_builder(void)
2004 GC_ASSERT(GC_mark_lock_holder == (unsigned long)GetCurrentThreadId());
2005 GC_ASSERT(builder_cv != 0);
2006 GC_ASSERT(GC_fl_builder_count == 0);
2007 if (SetEvent(builder_cv) == FALSE)
2008 ABORT("SetEvent() failed");
2011 # ifdef DONT_USE_SIGNALANDWAIT
2013 /* mark_cv is used (for waiting) by a non-helper thread. */
2015 GC_INNER void GC_wait_marker(void)
2017 HANDLE event = mark_cv;
2018 DWORD thread_id = GetCurrentThreadId();
2019 int i = (int)GC_markers - 1;
2021 if (GC_marker_Id[i] == thread_id) {
2022 event = GC_marker_cv[i];
2027 if (ResetEvent(event) == FALSE)
2028 ABORT("ResetEvent() failed");
2029 GC_release_mark_lock();
2030 if (WaitForSingleObject(event, INFINITE) == WAIT_FAILED)
2031 ABORT("WaitForSingleObject() failed");
2032 GC_acquire_mark_lock();
2035 GC_INNER void GC_notify_all_marker(void)
2037 DWORD thread_id = GetCurrentThreadId();
2038 int i = (int)GC_markers - 1;
2040 /* Notify every marker ignoring self (for efficiency). */
2041 if (SetEvent(GC_marker_Id[i] != thread_id ? GC_marker_cv[i] :
2043 ABORT("SetEvent() failed");
2047 # else /* DONT_USE_SIGNALANDWAIT */
2049 /* For GC_wait_marker/GC_notify_all_marker() the above technique */
2050 /* does not work because they are used with different checked */
2051 /* conditions in different places (and, in addition, notifying is */
2052 /* done after leaving critical section) and this could result in */
2053 /* a signal loosing between checking for a particular condition */
2054 /* and calling WaitForSingleObject. So, we use PulseEvent() and */
2055 /* NT SignalObjectAndWait() (which atomically sets mutex event to */
2056 /* signaled state and starts waiting on condvar). A special */
2057 /* case here is GC_mark_mutex_waitcnt == 1 (i.e. nobody waits for */
2058 /* mark lock at this moment) - we don't change it (otherwise we */
2059 /* may loose a signal sent between decrementing */
2060 /* GC_mark_mutex_waitcnt and calling WaitForSingleObject()). */
2063 /* SignalObjectAndWait() is missing in WinCE (for now), so you */
2064 /* should supply its emulation (externally) to use this code. */
2065 WINBASEAPI DWORD WINAPI SignalObjectAndWait(HANDLE, HANDLE, DWORD,
2067 # define signalObjectAndWait_func SignalObjectAndWait
2069 typedef DWORD (WINAPI * SignalObjectAndWait_type)(HANDLE, HANDLE,
2071 static SignalObjectAndWait_type signalObjectAndWait_func = 0;
2074 GC_INNER void GC_wait_marker(void)
2076 /* Here we assume that GC_wait_marker() is always called */
2077 /* from a while(check_cond) loop. */
2079 GC_ASSERT(mark_cv != 0);
2081 /* We inline GC_release_mark_lock() to have atomic */
2082 /* unlock-and-wait action here. */
2083 GC_ASSERT(GC_mark_lock_holder == (unsigned long)GetCurrentThreadId());
2084 # ifdef GC_ASSERTIONS
2085 GC_mark_lock_holder = NO_THREAD;
2088 if ((waitcnt = AO_load(&GC_mark_mutex_waitcnt)) > 1) {
2089 (void)AO_fetch_and_sub1_release(&GC_mark_mutex_waitcnt);
2091 GC_ASSERT(AO_load(&GC_mark_mutex_waitcnt) != 0);
2094 /* The state of mark_cv is non-signaled here. */
2095 if (signalObjectAndWait_func(mark_mutex_event /* hObjectToSignal */,
2096 mark_cv /* hObjectToWaitOn */,
2097 INFINITE /* timeout */,
2098 FALSE /* isAlertable */) == WAIT_FAILED)
2099 ABORT("SignalObjectAndWait() failed");
2100 /* The state of mark_cv is non-signaled here again. */
2103 GC_acquire_mark_lock();
2105 GC_ASSERT(GC_mark_mutex_waitcnt != 0);
2106 /* Acquire mark lock */
2107 if (WaitForSingleObject(mark_mutex_event, INFINITE) == WAIT_FAILED)
2108 ABORT("WaitForSingleObject() failed");
2109 GC_ASSERT(GC_mark_lock_holder == NO_THREAD);
2110 # ifdef GC_ASSERTIONS
2111 GC_mark_lock_holder = (unsigned long)GetCurrentThreadId();
2116 GC_INNER void GC_notify_all_marker(void)
2118 GC_ASSERT(mark_cv != 0);
2119 if (PulseEvent(mark_cv) == FALSE)
2120 ABORT("PulseEvent() failed");
2123 # endif /* !DONT_USE_SIGNALANDWAIT */
2125 # endif /* ! GC_PTHREADS_PARAMARK */
2127 #endif /* PARALLEL_MARK */
2131 /* We have no DllMain to take care of new threads. Thus we */
2132 /* must properly intercept thread creation. */
2135 LPTHREAD_START_ROUTINE start;
2139 STATIC void * GC_CALLBACK GC_win32_start_inner(struct GC_stack_base *sb,
2143 LPTHREAD_START_ROUTINE start = ((thread_args *)arg)->start;
2144 LPVOID param = ((thread_args *)arg)->param;
2146 GC_register_my_thread(sb); /* This waits for an in-progress GC. */
2148 # ifdef DEBUG_THREADS
2149 GC_log_printf("thread 0x%lx starting...\n", (long)GetCurrentThreadId());
2154 /* Clear the thread entry even if we exit with an exception. */
2155 /* This is probably pointless, since an uncaught exception is */
2156 /* supposed to result in the process being killed. */
2161 ret = (void *)(word)(*start)(param);
2167 GC_unregister_my_thread();
2170 # ifdef DEBUG_THREADS
2171 GC_log_printf("thread 0x%lx returned from start routine\n",
2172 (long)GetCurrentThreadId());
2177 STATIC DWORD WINAPI GC_win32_start(LPVOID arg)
2179 return (DWORD)(word)GC_call_with_stack_base(GC_win32_start_inner, arg);
2182 GC_API HANDLE WINAPI GC_CreateThread(
2183 LPSECURITY_ATTRIBUTES lpThreadAttributes,
2185 LPTHREAD_START_ROUTINE lpStartAddress,
2186 LPVOID lpParameter, DWORD dwCreationFlags,
2192 if (!parallel_initialized) GC_init_parallel();
2193 /* make sure GC is initialized (i.e. main thread is */
2194 /* attached, tls initialized). */
2196 # ifdef DEBUG_THREADS
2197 GC_log_printf("About to create a thread from 0x%lx\n",
2198 (long)GetCurrentThreadId());
2200 if (GC_win32_dll_threads) {
2201 return CreateThread(lpThreadAttributes, dwStackSize, lpStartAddress,
2202 lpParameter, dwCreationFlags, lpThreadId);
2204 args = GC_malloc_uncollectable(sizeof(thread_args));
2205 /* Handed off to and deallocated by child thread. */
2207 SetLastError(ERROR_NOT_ENOUGH_MEMORY);
2211 /* set up thread arguments */
2212 args -> start = lpStartAddress;
2213 args -> param = lpParameter;
2215 GC_need_to_lock = TRUE;
2216 thread_h = CreateThread(lpThreadAttributes, dwStackSize, GC_win32_start,
2217 args, dwCreationFlags, lpThreadId);
2218 if (thread_h == 0) GC_free(args);
2223 GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread(DWORD dwExitCode)
2225 GC_unregister_my_thread();
2226 ExitThread(dwExitCode);
2231 GC_API GC_uintptr_t GC_CALL GC_beginthreadex(
2232 void *security, unsigned stack_size,
2233 unsigned (__stdcall *start_address)(void *),
2234 void *arglist, unsigned initflag,
2237 GC_uintptr_t thread_h;
2240 if (!parallel_initialized) GC_init_parallel();
2241 /* make sure GC is initialized (i.e. main thread is */
2242 /* attached, tls initialized). */
2243 # ifdef DEBUG_THREADS
2244 GC_log_printf("About to create a thread from 0x%lx\n",
2245 (long)GetCurrentThreadId());
2248 if (GC_win32_dll_threads) {
2249 return _beginthreadex(security, stack_size, start_address,
2250 arglist, initflag, thrdaddr);
2252 args = GC_malloc_uncollectable(sizeof(thread_args));
2253 /* Handed off to and deallocated by child thread. */
2255 /* MSDN docs say _beginthreadex() returns 0 on error and sets */
2256 /* errno to either EAGAIN (too many threads) or EINVAL (the */
2257 /* argument is invalid or the stack size is incorrect), so we */
2258 /* set errno to EAGAIN on "not enough memory". */
2263 /* set up thread arguments */
2264 args -> start = (LPTHREAD_START_ROUTINE)start_address;
2265 args -> param = arglist;
2267 GC_need_to_lock = TRUE;
2268 thread_h = _beginthreadex(security, stack_size,
2269 (unsigned (__stdcall *)(void *))GC_win32_start,
2270 args, initflag, thrdaddr);
2271 if (thread_h == 0) GC_free(args);
2276 GC_API void GC_CALL GC_endthreadex(unsigned retval)
2278 GC_unregister_my_thread();
2279 _endthreadex(retval);
2282 # endif /* !MSWINCE */
2284 #endif /* !GC_PTHREADS */
2286 #ifdef GC_WINMAIN_REDIRECT
2287 /* This might be useful on WinCE. Shouldn't be used with GC_DLL. */
2289 # if defined(MSWINCE) && defined(UNDER_CE)
2290 # define WINMAIN_LPTSTR LPWSTR
2292 # define WINMAIN_LPTSTR LPSTR
2295 /* This is defined in gc.h. */
2298 /* Defined outside GC by an application. */
2299 int WINAPI GC_WinMain(HINSTANCE, HINSTANCE, WINMAIN_LPTSTR, int);
2302 HINSTANCE hInstance;
2303 HINSTANCE hPrevInstance;
2304 WINMAIN_LPTSTR lpCmdLine;
2308 static DWORD WINAPI main_thread_start(LPVOID arg)
2310 main_thread_args * args = (main_thread_args *) arg;
2311 return (DWORD)GC_WinMain(args->hInstance, args->hPrevInstance,
2312 args->lpCmdLine, args->nShowCmd);
2315 STATIC void * GC_waitForSingleObjectInfinite(void * handle)
2317 return (void *)(word)WaitForSingleObject((HANDLE)handle, INFINITE);
2320 # ifndef WINMAIN_THREAD_STACK_SIZE
2321 # define WINMAIN_THREAD_STACK_SIZE 0 /* default value */
2324 int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
2325 WINMAIN_LPTSTR lpCmdLine, int nShowCmd)
2327 DWORD exit_code = 1;
2329 main_thread_args args = {
2330 hInstance, hPrevInstance, lpCmdLine, nShowCmd
2335 /* initialize everything */
2338 /* start the main thread */
2339 thread_h = GC_CreateThread(NULL /* lpsa */,
2340 WINMAIN_THREAD_STACK_SIZE /* ignored on WinCE */,
2341 main_thread_start, &args, 0 /* fdwCreate */,
2344 if (thread_h != NULL) {
2345 if ((DWORD)(word)GC_do_blocking(GC_waitForSingleObjectInfinite,
2346 (void *)thread_h) == WAIT_FAILED)
2347 ABORT("WaitForSingleObject(main_thread) failed");
2348 GetExitCodeThread (thread_h, &exit_code);
2349 CloseHandle (thread_h);
2351 ABORT("GC_CreateThread(main_thread) failed");
2356 DeleteCriticalSection(&GC_allocate_ml);
2358 return (int) exit_code;
2361 #endif /* GC_WINMAIN_REDIRECT */
2363 /* Called by GC_init() - we hold the allocation lock. */
2364 GC_INNER void GC_thr_init(void)
2366 struct GC_stack_base sb;
2367 # ifdef GC_ASSERTIONS
2371 GC_ASSERT(I_HOLD_LOCK());
2372 if (GC_thr_initialized) return;
2373 GC_main_thread = GetCurrentThreadId();
2374 GC_thr_initialized = TRUE;
2376 # ifdef CAN_HANDLE_FORK
2377 /* Prepare for forks if requested. */
2379 && pthread_atfork(GC_fork_prepare_proc, GC_fork_parent_proc,
2380 GC_fork_child_proc) != 0)
2381 ABORT("pthread_atfork failed");
2384 /* Add the initial thread, so we can stop it. */
2385 # ifdef GC_ASSERTIONS
2388 GC_get_stack_base(&sb);
2389 GC_ASSERT(sb_result == GC_SUCCESS);
2391 # if defined(PARALLEL_MARK)
2392 /* Set GC_markers. */
2394 char * markers_string = GETENV("GC_MARKERS");
2395 if (markers_string != NULL) {
2396 GC_markers = atoi(markers_string);
2397 if (GC_markers > MAX_MARKERS) {
2398 WARN("Limiting number of mark threads\n", 0);
2399 GC_markers = MAX_MARKERS;
2403 /* There is no GetProcessAffinityMask() in WinCE. */
2404 /* GC_sysinfo is already initialized. */
2405 GC_markers = GC_sysinfo.dwNumberOfProcessors;
2408 DWORD_PTR procMask = 0;
2415 if (GetProcessAffinityMask(GetCurrentProcess(),
2416 (void *)&procMask, (void *)&sysMask)
2420 } while ((procMask &= procMask - 1) != 0);
2424 # ifdef GC_MIN_MARKERS
2425 /* This is primarily for testing on systems without getenv(). */
2426 if (GC_markers < GC_MIN_MARKERS)
2427 GC_markers = GC_MIN_MARKERS;
2429 if (GC_markers >= MAX_MARKERS)
2430 GC_markers = MAX_MARKERS; /* silently limit GC_markers value */
2434 /* Set GC_parallel. */
2436 # if !defined(GC_PTHREADS_PARAMARK) && !defined(MSWINCE) \
2437 && !defined(DONT_USE_SIGNALANDWAIT)
2439 /* SignalObjectAndWait() API call works only under NT. */
2441 if (GC_win32_dll_threads || GC_markers <= 1
2442 # if !defined(GC_PTHREADS_PARAMARK) && !defined(MSWINCE) \
2443 && !defined(DONT_USE_SIGNALANDWAIT)
2445 || (hK32 = GetModuleHandle(TEXT("kernel32.dll"))) == (HMODULE)0
2446 || (signalObjectAndWait_func = (SignalObjectAndWait_type)
2447 GetProcAddress(hK32, "SignalObjectAndWait")) == 0
2450 /* Disable parallel marking. */
2451 GC_parallel = FALSE;
2454 # ifndef GC_PTHREADS_PARAMARK
2455 /* Initialize Win32 event objects for parallel marking. */
2456 mark_mutex_event = CreateEvent(NULL /* attrs */,
2457 FALSE /* isManualReset */,
2458 FALSE /* initialState */, NULL /* name */);
2459 builder_cv = CreateEvent(NULL /* attrs */,
2460 TRUE /* isManualReset */,
2461 FALSE /* initialState */, NULL /* name */);
2462 mark_cv = CreateEvent(NULL /* attrs */, TRUE /* isManualReset */,
2463 FALSE /* initialState */, NULL /* name */);
2464 if (mark_mutex_event == (HANDLE)0 || builder_cv == (HANDLE)0
2465 || mark_cv == (HANDLE)0)
2466 ABORT("CreateEvent() failed");
2469 /* Disable true incremental collection, but generational is OK. */
2470 GC_time_limit = GC_TIME_UNLIMITED;
2473 # endif /* PARALLEL_MARK */
2475 GC_ASSERT(0 == GC_lookup_thread_inner(GC_main_thread));
2476 GC_register_my_thread_inner(&sb, GC_main_thread);
2478 # ifdef PARALLEL_MARK
2479 /* If we are using a parallel marker, actually start helper threads. */
2480 if (GC_parallel) start_mark_threads();
2481 if (GC_print_stats) {
2482 GC_log_printf("Started %ld mark helper threads\n", GC_markers - 1);
2490 void *(*start_routine)(void *);
2495 GC_API int GC_pthread_join(pthread_t pthread_id, void **retval)
2500 # ifdef DEBUG_THREADS
2501 GC_log_printf("thread %p(0x%lx) is joining thread %p\n",
2502 GC_PTHREAD_PTRVAL(pthread_self()),
2503 (long)GetCurrentThreadId(), GC_PTHREAD_PTRVAL(pthread_id));
2506 if (!parallel_initialized) GC_init_parallel();
2508 /* Thread being joined might not have registered itself yet. */
2509 /* After the join,thread id may have been recycled. */
2510 /* FIXME: It would be better if this worked more like */
2511 /* pthread_support.c. */
2512 # ifndef GC_WIN32_PTHREADS
2513 while ((t = GC_lookup_pthread(pthread_id)) == 0)
2517 result = pthread_join(pthread_id, retval);
2519 # ifdef GC_WIN32_PTHREADS
2520 /* win32_pthreads id are unique */
2521 t = GC_lookup_pthread(pthread_id);
2524 if (!GC_win32_dll_threads) {
2528 GC_delete_gc_thread(t);
2530 } /* otherwise DllMain handles it. */
2532 # ifdef DEBUG_THREADS
2533 GC_log_printf("thread %p(0x%lx) completed join with thread %p\n",
2534 GC_PTHREAD_PTRVAL(pthread_self()),
2535 (long)GetCurrentThreadId(), GC_PTHREAD_PTRVAL(pthread_id));
2540 /* Cygwin-pthreads calls CreateThread internally, but it's not easily */
2541 /* interceptible by us..., so intercept pthread_create instead. */
2542 GC_API int GC_pthread_create(pthread_t *new_thread,
2543 GC_PTHREAD_CREATE_CONST pthread_attr_t *attr,
2544 void *(*start_routine)(void *), void *arg)
2546 if (!parallel_initialized) GC_init_parallel();
2547 /* make sure GC is initialized (i.e. main thread is attached) */
2548 if (GC_win32_dll_threads) {
2549 return pthread_create(new_thread, attr, start_routine, arg);
2552 struct start_info * si;
2554 /* This is otherwise saved only in an area mmapped by the thread */
2555 /* library, which isn't visible to the collector. */
2556 si = GC_malloc_uncollectable(sizeof(struct start_info));
2557 if (0 == si) return(EAGAIN);
2559 si -> start_routine = start_routine;
2562 pthread_attr_getdetachstate(attr, &si->detached)
2563 == PTHREAD_CREATE_DETACHED) {
2564 si->detached = TRUE;
2567 # ifdef DEBUG_THREADS
2568 GC_log_printf("About to create a thread from %p(0x%lx)\n",
2569 GC_PTHREAD_PTRVAL(pthread_self()),
2570 (long)GetCurrentThreadId());
2572 GC_need_to_lock = TRUE;
2573 result = pthread_create(new_thread, attr, GC_pthread_start, si);
2575 if (result) { /* failure */
2582 STATIC void * GC_CALLBACK GC_pthread_start_inner(struct GC_stack_base *sb,
2585 struct start_info * si = arg;
2587 void *(*start)(void *);
2589 DWORD thread_id = GetCurrentThreadId();
2590 pthread_t pthread_id = pthread_self();
2594 # ifdef DEBUG_THREADS
2595 GC_log_printf("thread %p(0x%x) starting...\n",
2596 GC_PTHREAD_PTRVAL(pthread_id), (int)thread_id);
2599 GC_ASSERT(!GC_win32_dll_threads);
2600 /* If a GC occurs before the thread is registered, that GC will */
2601 /* ignore this thread. That's fine, since it will block trying to */
2602 /* acquire the allocation lock, and won't yet hold interesting */
2605 /* We register the thread here instead of in the parent, so that */
2606 /* we don't need to hold the allocation lock during pthread_create. */
2607 me = GC_register_my_thread_inner(sb, thread_id);
2608 SET_PTHREAD_MAP_CACHE(pthread_id, thread_id);
2609 me -> pthread_id = pthread_id;
2610 if (si->detached) me -> flags |= DETACHED;
2613 start = si -> start_routine;
2614 start_arg = si -> arg;
2616 GC_free(si); /* was allocated uncollectable */
2618 pthread_cleanup_push(GC_thread_exit_proc, (void *)me);
2619 result = (*start)(start_arg);
2620 me -> status = result;
2621 pthread_cleanup_pop(1);
2623 # ifdef DEBUG_THREADS
2624 GC_log_printf("thread %p(0x%x) returned from start routine\n",
2625 GC_PTHREAD_PTRVAL(pthread_id), (int)thread_id);
2630 STATIC void * GC_pthread_start(void * arg)
2632 return GC_call_with_stack_base(GC_pthread_start_inner, arg);
2635 STATIC void GC_thread_exit_proc(void *arg)
2637 GC_thread me = (GC_thread)arg;
2640 GC_ASSERT(!GC_win32_dll_threads);
2641 # ifdef DEBUG_THREADS
2642 GC_log_printf("thread %p(0x%lx) called pthread_exit()\n",
2643 GC_PTHREAD_PTRVAL(pthread_self()),
2644 (long)GetCurrentThreadId());
2648 GC_wait_for_gc_completion(FALSE);
2649 # if defined(THREAD_LOCAL_ALLOC)
2650 GC_destroy_thread_local(&(me->tlfs));
2652 if (me -> flags & DETACHED) {
2653 GC_delete_thread(GetCurrentThreadId());
2655 /* deallocate it as part of join */
2656 me -> flags |= FINISHED;
2661 # ifndef GC_NO_PTHREAD_SIGMASK
2662 /* Win32 pthread does not support sigmask. */
2663 /* So, nothing required here... */
2664 GC_API int GC_pthread_sigmask(int how, const sigset_t *set,
2667 if (!parallel_initialized) GC_init_parallel();
2668 return pthread_sigmask(how, set, oset);
2670 # endif /* !GC_NO_PTHREAD_SIGMASK */
2672 GC_API int GC_pthread_detach(pthread_t thread)
2678 if (!parallel_initialized) GC_init_parallel();
2680 t = GC_lookup_pthread(thread);
2682 result = pthread_detach(thread);
2685 t -> flags |= DETACHED;
2686 /* Here the pthread thread id may have been recycled. */
2687 if ((t -> flags & FINISHED) != 0) {
2688 GC_delete_gc_thread(t);
2695 #else /* !GC_PTHREADS */
2697 # ifndef GC_NO_THREADS_DISCOVERY
2698 /* We avoid acquiring locks here, since this doesn't seem to be */
2699 /* preemptible. This may run with an uninitialized collector, in */
2700 /* which case we don't do much. This implies that no threads other */
2701 /* than the main one should be created with an uninitialized */
2702 /* collector. (The alternative of initializing the collector here */
2703 /* seems dangerous, since DllMain is limited in what it can do.) */
2706 BOOL WINAPI DllMain(HINSTANCE inst, ULONG reason, LPVOID reserved)
2708 struct GC_stack_base sb;
2710 # ifdef GC_ASSERTIONS
2713 static int entry_count = 0;
2715 if (!GC_win32_dll_threads && parallel_initialized) return TRUE;
2718 case DLL_THREAD_ATTACH:
2719 # ifdef PARALLEL_MARK
2720 /* Don't register marker threads. */
2722 /* We could reach here only if parallel_initialized == FALSE. */
2726 GC_ASSERT(entry_count == 0 || parallel_initialized);
2727 ++entry_count; /* and fall through: */
2728 case DLL_PROCESS_ATTACH:
2729 /* This may run with the collector uninitialized. */
2730 thread_id = GetCurrentThreadId();
2731 if (parallel_initialized && GC_main_thread != thread_id) {
2732 /* Don't lock here. */
2733 # ifdef GC_ASSERTIONS
2736 GC_get_stack_base(&sb);
2737 GC_ASSERT(sb_result == GC_SUCCESS);
2738 # if defined(THREAD_LOCAL_ALLOC) || defined(PARALLEL_MARK)
2739 ABORT("Cannot initialize thread local cache from DllMain");
2741 GC_register_my_thread_inner(&sb, thread_id);
2742 } /* o.w. we already did it during GC_thr_init, called by GC_init */
2745 case DLL_THREAD_DETACH:
2746 /* We are hopefully running in the context of the exiting thread. */
2747 GC_ASSERT(parallel_initialized);
2748 if (GC_win32_dll_threads) {
2749 GC_delete_thread(GetCurrentThreadId());
2753 case DLL_PROCESS_DETACH:
2754 if (GC_win32_dll_threads) {
2756 int my_max = (int)GC_get_max_thread_index();
2758 for (i = 0; i <= my_max; ++i) {
2759 if (AO_load(&(dll_thread_table[i].tm.in_use)))
2760 GC_delete_gc_thread(dll_thread_table + i);
2763 DeleteCriticalSection(&GC_allocate_ml);
2769 # endif /* !GC_NO_THREADS_DISCOVERY */
2771 #endif /* !GC_PTHREADS */
2773 /* Perform all initializations, including those that */
2774 /* may require allocation. */
2775 /* Called without allocation lock. */
2776 /* Must be called before a second thread is created. */
2777 GC_INNER void GC_init_parallel(void)
2779 # if defined(THREAD_LOCAL_ALLOC)
2784 if (parallel_initialized) return;
2785 parallel_initialized = TRUE;
2786 /* GC_init() calls us back, so set flag first. */
2788 if (!GC_is_initialized) GC_init();
2789 if (GC_win32_dll_threads) {
2790 GC_need_to_lock = TRUE;
2791 /* Cannot intercept thread creation. Hence we don't know if */
2792 /* other threads exist. However, client is not allowed to */
2793 /* create other threads before collector initialization. */
2794 /* Thus it's OK not to lock before this. */
2796 /* Initialize thread local free lists if used. */
2797 # if defined(THREAD_LOCAL_ALLOC)
2799 me = GC_lookup_thread_inner(GetCurrentThreadId());
2800 CHECK_LOOKUP_MY_THREAD(me);
2801 GC_init_thread_local(&me->tlfs);
2806 #if defined(USE_PTHREAD_LOCKS)
2807 /* Support for pthread locking code. */
2808 /* Pthread_mutex_try_lock may not win here, */
2809 /* due to builtin support for spinning first? */
2811 GC_INNER volatile GC_bool GC_collecting = 0;
2812 /* A hint that we're in the collector and */
2813 /* holding the allocation lock for an */
2814 /* extended period. */
2816 GC_INNER void GC_lock(void)
2818 pthread_mutex_lock(&GC_allocate_ml);
2820 #endif /* USE_PTHREAD_LOCKS */
2822 #if defined(THREAD_LOCAL_ALLOC)
2824 /* Add thread-local allocation support. VC++ uses __declspec(thread). */
2826 /* We must explicitly mark ptrfree and gcj free lists, since the free */
2827 /* list links wouldn't otherwise be found. We also set them in the */
2828 /* normal free lists, since that involves touching less memory than if */
2829 /* we scanned them normally. */
2830 GC_INNER void GC_mark_thread_local_free_lists(void)
2835 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
2836 for (p = GC_threads[i]; 0 != p; p = p -> tm.next) {
2837 if (!KNOWN_FINISHED(p)) {
2838 # ifdef DEBUG_THREADS
2839 GC_log_printf("Marking thread locals for 0x%x\n", (int)p -> id);
2841 GC_mark_thread_local_fls_for(&(p->tlfs));
2847 # if defined(GC_ASSERTIONS)
2848 void GC_check_tls_for(GC_tlfs p);
2849 # if defined(USE_CUSTOM_SPECIFIC)
2850 void GC_check_tsd_marks(tsd *key);
2852 /* Check that all thread-local free-lists are completely marked. */
2853 /* also check that thread-specific-data structures are marked. */
2854 void GC_check_tls(void)
2859 for (i = 0; i < THREAD_TABLE_SZ; ++i) {
2860 for (p = GC_threads[i]; 0 != p; p = p -> tm.next) {
2861 if (!KNOWN_FINISHED(p))
2862 GC_check_tls_for(&(p->tlfs));
2865 # if defined(USE_CUSTOM_SPECIFIC)
2866 if (GC_thread_key != 0)
2867 GC_check_tsd_marks(GC_thread_key);
2870 # endif /* GC_ASSERTIONS */
2872 #endif /* THREAD_LOCAL_ALLOC ... */
2874 # ifndef GC_NO_THREAD_REDIRECTS
2875 /* Restore thread calls redirection. */
2876 # define CreateThread GC_CreateThread
2877 # define ExitThread GC_ExitThread
2878 # undef _beginthreadex
2879 # define _beginthreadex GC_beginthreadex
2880 # undef _endthreadex
2881 # define _endthreadex GC_endthreadex
2882 # endif /* !GC_NO_THREAD_REDIRECTS */
2884 #endif /* GC_WIN32_THREADS */