* Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
+/*
+ * SGen cannot deal with invalid pointers on the heap or in registered roots. Sometimes we
+ * need to copy or zero out memory in code that might be interrupted by collections. To
+ * guarantee that those operations will not result in invalid pointers, we must do it
+ * word-atomically.
+ *
+ * libc's bzero() and memcpy()/memmove() functions do not guarantee word-atomicity, even in
+ * cases where one would assume so. For instance, some implementations (like Darwin's on
+ * x86) have variants of memcpy() using vector instructions. Those may copy bytewise for
+ * the region preceding the first vector-aligned address. That region could be
+ * word-aligned, but it would still be copied byte-wise.
+ *
+ * All our memory writes here are to "volatile" locations. This is so that C compilers
+ * don't "optimize" our code back to calls to bzero()/memmove(). LLVM, specifically, will
+ * do that.
+ */
+
#include <config.h>
#include "metadata/gc-internal.h"
#endif
#define BZERO_WORDS(dest,words) do { \
- void **__d = (void**)(dest); \
+ void * volatile *__d = (void* volatile*)(dest); \
int __n = (words); \
int __i; \
for (__i = 0; __i < __n; ++__i) \
__d [__i] = NULL; \
} while (0)
+
/**
- * mono_gc_bzero:
+ * mono_gc_bzero_aligned:
* @dest: address to start to clear
* @size: size of the region to clear
*
* Zero @size bytes starting at @dest.
- *
- * Use this to zero memory that can hold managed pointers.
+ * The address of @dest MUST be aligned to word boundaries
*
* FIXME borrow faster code from some BSD libc or bionic
*/
void
-mono_gc_bzero (void *dest, size_t size)
+mono_gc_bzero_aligned (void *dest, size_t size)
{
- char *d = (char*)dest;
+ volatile char *d = (char*)dest;
size_t tail_bytes, word_bytes;
- /*
- If we're copying less than a word, just use memset.
-
- We cannot bail out early if both are aligned because some implementations
- use byte copying for sizes smaller than 16. OSX, on this case.
- */
- if (size < sizeof(void*)) {
- memset (dest, 0, size);
- return;
- }
-
- /*align to word boundary */
- while (unaligned_bytes (d) && size) {
- *d++ = 0;
- --size;
- }
+ g_assert (unaligned_bytes (dest) == 0);
/* copy all words with memmove */
word_bytes = (size_t)align_down (size);
}
}
+/**
+ * mono_gc_bzero_atomic:
+ * @dest: address to start to clear
+ * @size: size of the region to clear
+ *
+ * Zero @size bytes starting at @dest.
+ *
+ * Use this to zero memory without word tearing when dest is aligned.
+ */
+void
+mono_gc_bzero_atomic (void *dest, size_t size)
+{
+ if (unaligned_bytes (dest))
+ memset (dest, 0, size);
+ else
+ mono_gc_bzero_aligned (dest, size);
+}
+
#define MEMMOVE_WORDS_UPWARD(dest,src,words) do { \
- void **__d = (void**)(dest); \
+ void * volatile *__d = (void* volatile*)(dest); \
void **__s = (void**)(src); \
int __n = (int)(words); \
int __i; \
} while (0)
#define MEMMOVE_WORDS_DOWNWARD(dest,src,words) do { \
- void **__d = (void**)(dest); \
+ void * volatile *__d = (void* volatile*)(dest); \
void **__s = (void**)(src); \
int __n = (int)(words); \
int __i; \
__d [__i] = __s [__i]; \
} while (0)
+
/**
- * mono_gc_memmove:
+ * mono_gc_memmove_aligned:
* @dest: destination of the move
* @src: source
* @size: size of the block to move
*
* Move @size bytes from @src to @dest.
- * size MUST be a multiple of sizeof (gpointer)
*
- */
-void
-mono_gc_memmove (void *dest, const void *src, size_t size)
+ * Use this to copy memory without word tearing when both pointers are aligned
+ */void
+mono_gc_memmove_aligned (void *dest, const void *src, size_t size)
{
+ g_assert (unaligned_bytes (dest) == 0);
+ g_assert (unaligned_bytes (src) == 0);
+
/*
If we're copying less than a word we don't need to worry about word tearing
so we bailout to memmove early.
* using memmove, which must handle it.
*/
if (dest > src && ((size_t)((char*)dest - (char*)src) < size)) { /*backward copy*/
- char *p = (char*)dest + size;
+ volatile char *p = (char*)dest + size;
char *s = (char*)src + size;
char *start = (char*)dest;
char *align_end = MAX((char*)dest, (char*)align_down (p));
size_t bytes_to_memmove;
while (p > align_end)
- *--p = *--s;
+ *--p = *--s;
word_start = align_up (start);
bytes_to_memmove = p - word_start;
p -= bytes_to_memmove;
s -= bytes_to_memmove;
MEMMOVE_WORDS_DOWNWARD (p, s, bytes_to_words (bytes_to_memmove));
-
- while (p > start)
- *--p = *--s;
} else {
- char *d = (char*)dest;
+ volatile char *d = (char*)dest;
const char *s = (const char*)src;
size_t tail_bytes;
- /*align to word boundary */
- while (unaligned_bytes (d)) {
- *d++ = *s++;
- --size;
- }
-
/* copy all words with memmove */
MEMMOVE_WORDS_UPWARD (d, s, bytes_to_words (align_down (size)));
}
}
}
+
+/**
+ * mono_gc_memmove_atomic:
+ * @dest: destination of the move
+ * @src: source
+ * @size: size of the block to move
+ *
+ * Move @size bytes from @src to @dest.
+ *
+ * Use this to copy memory without word tearing when both pointers are aligned
+ */
+void
+mono_gc_memmove_atomic (void *dest, const void *src, size_t size)
+{
+ if (unaligned_bytes (_toi (dest) | _toi (src)))
+ memmove (dest, src, size);
+ else
+ mono_gc_memmove_aligned (dest, src, size);
+}