[sgen] Separate concurrent M&S object operations into start, concurrent, finish.

[mono.git] / mono / metadata / sgen-gray.h

diff --git a/mono/metadata/sgen-gray.h b/mono/metadata/sgen-gray.h
index 0b99456c0b8ddb2a2681826f6ab29676c00d8a7f..f222c55b9070885f20868f626a4bbb43c5d35561 100644 (file)
--- a/mono/metadata/sgen-gray.h
+++ b/mono/metadata/sgen-gray.h
@@ -20,6 +20,8 @@
 #ifndef __MONO_SGEN_GRAY_H__
 #define __MONO_SGEN_GRAY_H__
 
+#include "metadata/sgen-protocol.h"
+
 /*
  * This gray queue has to be as optimized as possible, because it is in the core of
  * the mark/copy phase of the garbage collector. The memory access has then to be as
@@ -48,7 +50,14 @@
  * array and another 1 for the actual value in the array.
  */
 
-#define SGEN_GRAY_QUEUE_SECTION_SIZE   (128 - 3)
+/* SGEN_GRAY_QUEUE_HEADER_SIZE is number of machine words */
+#ifdef SGEN_CHECK_GRAY_OBJECT_SECTIONS
+#define SGEN_GRAY_QUEUE_HEADER_SIZE    4
+#else
+#define SGEN_GRAY_QUEUE_HEADER_SIZE    2
+#endif
+
+#define SGEN_GRAY_QUEUE_SECTION_SIZE   (128 - SGEN_GRAY_QUEUE_HEADER_SIZE)
 
 #ifdef SGEN_CHECK_GRAY_OBJECT_SECTIONS
 typedef enum {
@@ -59,6 +68,14 @@ typedef enum {
 } GrayQueueSectionState;
 #endif
 
+typedef struct _GrayQueueEntry GrayQueueEntry;
+struct _GrayQueueEntry {
+       char *obj;
+       mword desc;
+};
+
+#define SGEN_GRAY_QUEUE_ENTRY(obj,desc)        { (obj), (desc) }
+
 /*
  * This is a stack now instead of a queue, so the most recently added items are removed
  * first, improving cache locality, and keeping the stack size manageable.
@@ -75,7 +92,7 @@ struct _GrayQueueSection {
 #endif
        int size;
        GrayQueueSection *next;
-       char *objects [SGEN_GRAY_QUEUE_SECTION_SIZE];
+       GrayQueueEntry entries [SGEN_GRAY_QUEUE_SECTION_SIZE];
 };
 
 typedef struct _SgenGrayQueue SgenGrayQueue;
@@ -84,7 +101,7 @@ typedef void (*GrayQueueAllocPrepareFunc) (SgenGrayQueue*);
 typedef void (*GrayQueueEnqueueCheckFunc) (char*);
 
 struct _SgenGrayQueue {
-       char **cursor;
+       GrayQueueEntry *cursor;
        GrayQueueSection *first;
        GrayQueueSection *free_list;
        GrayQueueAllocPrepareFunc alloc_prepare_func;
@@ -105,28 +122,42 @@ struct _SgenSectionGrayQueue {
 #endif
 };
 
-#define GRAY_LAST_CURSOR_POSITION(s) ((char**)(s)->objects + SGEN_GRAY_QUEUE_SECTION_SIZE - 1)
-#define GRAY_SECOND_TO_LAST_CURSOR_POSITION(s) ((char**)(s)->objects + SGEN_GRAY_QUEUE_SECTION_SIZE - 2)
-#define GRAY_FIRST_CURSOR_POSITION(s) ((char**)(s)->objects)
+#define GRAY_LAST_CURSOR_POSITION(s) ((s)->entries + SGEN_GRAY_QUEUE_SECTION_SIZE - 1)
+#define GRAY_FIRST_CURSOR_POSITION(s) ((s)->entries)
 
-void sgen_gray_object_enqueue (SgenGrayQueue *queue, char *obj) MONO_INTERNAL;
-char* sgen_gray_object_dequeue (SgenGrayQueue *queue) MONO_INTERNAL;
-GrayQueueSection* sgen_gray_object_dequeue_section (SgenGrayQueue *queue) MONO_INTERNAL;
-void sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section) MONO_INTERNAL;
-void sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func) MONO_INTERNAL;
-void sgen_gray_object_queue_init_invalid (SgenGrayQueue *queue) MONO_INTERNAL;
+#ifdef HEAVY_STATISTICS
+extern guint64 stat_gray_queue_section_alloc;
+extern guint64 stat_gray_queue_section_free;
+extern guint64 stat_gray_queue_enqueue_fast_path;
+extern guint64 stat_gray_queue_dequeue_fast_path;
+extern guint64 stat_gray_queue_enqueue_slow_path;
+extern guint64 stat_gray_queue_dequeue_slow_path;
+#endif
+
+void sgen_init_gray_queues (void);
+
+void sgen_gray_object_enqueue (SgenGrayQueue *queue, char *obj, mword desc);
+GrayQueueEntry sgen_gray_object_dequeue (SgenGrayQueue *queue);
+GrayQueueSection* sgen_gray_object_dequeue_section (SgenGrayQueue *queue);
+void sgen_gray_object_enqueue_section (SgenGrayQueue *queue, GrayQueueSection *section);
+void sgen_gray_object_queue_trim_free_list (SgenGrayQueue *queue);
+void sgen_gray_object_queue_init (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func);
+void sgen_gray_object_queue_init_invalid (SgenGrayQueue *queue);
+void sgen_gray_queue_set_alloc_prepare (SgenGrayQueue *queue, GrayQueueAllocPrepareFunc alloc_prepare_func, void *data);
 void sgen_gray_object_queue_init_with_alloc_prepare (SgenGrayQueue *queue, GrayQueueEnqueueCheckFunc enqueue_check_func,
-               GrayQueueAllocPrepareFunc func, void *data) MONO_INTERNAL;
-void sgen_gray_object_queue_deinit (SgenGrayQueue *queue) MONO_INTERNAL;
-void sgen_gray_object_queue_disable_alloc_prepare (SgenGrayQueue *queue) MONO_INTERNAL;
-void sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue) MONO_INTERNAL;
-void sgen_gray_object_free_queue_section (GrayQueueSection *section) MONO_INTERNAL;
+               GrayQueueAllocPrepareFunc func, void *data);
+void sgen_gray_object_queue_deinit (SgenGrayQueue *queue);
+void sgen_gray_object_queue_disable_alloc_prepare (SgenGrayQueue *queue);
+void sgen_gray_object_alloc_queue_section (SgenGrayQueue *queue);
+void sgen_gray_object_free_queue_section (GrayQueueSection *section);
 
 void sgen_section_gray_queue_init (SgenSectionGrayQueue *queue, gboolean locked,
-               GrayQueueEnqueueCheckFunc enqueue_check_func) MONO_INTERNAL;
-gboolean sgen_section_gray_queue_is_empty (SgenSectionGrayQueue *queue) MONO_INTERNAL;
-GrayQueueSection* sgen_section_gray_queue_dequeue (SgenSectionGrayQueue *queue) MONO_INTERNAL;
-void sgen_section_gray_queue_enqueue (SgenSectionGrayQueue *queue, GrayQueueSection *section) MONO_INTERNAL;
+               GrayQueueEnqueueCheckFunc enqueue_check_func);
+gboolean sgen_section_gray_queue_is_empty (SgenSectionGrayQueue *queue);
+GrayQueueSection* sgen_section_gray_queue_dequeue (SgenSectionGrayQueue *queue);
+void sgen_section_gray_queue_enqueue (SgenSectionGrayQueue *queue, GrayQueueSection *section);
+
+gboolean sgen_gray_object_fill_prefetch (SgenGrayQueue *queue);
 
 static inline gboolean
 sgen_gray_object_queue_is_empty (SgenGrayQueue *queue)
@@ -134,4 +165,55 @@ sgen_gray_object_queue_is_empty (SgenGrayQueue *queue)
        return queue->first == NULL;
 }
 
+static inline MONO_ALWAYS_INLINE void
+GRAY_OBJECT_ENQUEUE (SgenGrayQueue *queue, char* obj, mword desc)
+{
+#if SGEN_MAX_DEBUG_LEVEL >= 9
+       sgen_gray_object_enqueue (queue, obj, desc);
+#else
+       if (G_UNLIKELY (!queue->first || queue->cursor == GRAY_LAST_CURSOR_POSITION (queue->first))) {
+               sgen_gray_object_enqueue (queue, obj, desc);
+       } else {
+               GrayQueueEntry entry = SGEN_GRAY_QUEUE_ENTRY (obj, desc);
+
+               HEAVY_STAT (stat_gray_queue_enqueue_fast_path ++);
+
+               *++queue->cursor = entry;
+#ifdef SGEN_HEAVY_BINARY_PROTOCOL
+               binary_protocol_gray_enqueue (queue, queue->cursor, obj);
+#endif
+       }
+#endif
+}
+
+static inline MONO_ALWAYS_INLINE void
+GRAY_OBJECT_DEQUEUE (SgenGrayQueue *queue, char** obj, mword *desc)
+{
+       GrayQueueEntry entry;
+#if SGEN_MAX_DEBUG_LEVEL >= 9
+       entry = sgen_gray_object_dequeue (queue);
+       *obj = entry.obj;
+       *desc = entry.desc;
+#else
+       if (!queue->first) {
+               HEAVY_STAT (stat_gray_queue_dequeue_fast_path ++);
+
+               *obj = NULL;
+       } else if (G_UNLIKELY (queue->cursor == GRAY_FIRST_CURSOR_POSITION (queue->first))) {
+               entry = sgen_gray_object_dequeue (queue);
+               *obj = entry.obj;
+               *desc = entry.desc;
+       } else {
+               HEAVY_STAT (stat_gray_queue_dequeue_fast_path ++);
+
+               entry = *queue->cursor--;
+               *obj = entry.obj;
+               *desc = entry.desc;
+#ifdef SGEN_HEAVY_BINARY_PROTOCOL
+               binary_protocol_gray_dequeue (queue, queue->cursor + 1, *obj);
+#endif
+       }
+#endif
+}
+
 #endif