12 #define USE_DL_PREFIX 1
14 #include "mono-codeman.h"
15 #include "mono-mmap.h"
16 #include "mono-counters.h"
18 #include <mono/metadata/class-internals.h>
19 #include <mono/metadata/profiler-private.h>
20 #ifdef HAVE_VALGRIND_MEMCHECK_H
21 #include <valgrind/memcheck.h>
24 #if defined(__native_client_codegen__) && defined(__native_client__)
26 #include <nacl/nacl_dyncode.h>
27 #include <mono/mini/mini.h>
30 static uintptr_t code_memory_used = 0;
33 * AMD64 processors maintain icache coherency only for pages which are
34 * marked executable. Also, windows DEP requires us to obtain executable memory from
35 * malloc when using dynamic code managers. The system malloc can't do this so we use a
36 * slighly modified version of Doug Lea's Malloc package for this purpose:
37 * http://g.oswego.edu/dl/html/malloc.html
42 #if defined(__ia64__) || defined(__x86_64__)
44 * We require 16 byte alignment on amd64 so the fp literals embedded in the code are
45 * properly aligned for SSE2.
51 #ifdef __native_client_codegen__
52 /* For Google Native Client, all targets of indirect control flow need to */
53 /* be aligned to bundle boundary. 16 bytes on ARM, 32 bytes on x86.
54 * MIN_ALIGN was updated to force alignment for calls from
55 * tramp-<arch>.c to mono_global_codeman_reserve() */
56 /* and mono_domain_code_reserve(). */
58 #define MIN_ALIGN kNaClBundleSize
62 /* if a chunk has less than this amount of free space it's considered full */
63 #define MAX_WASTAGE 32
67 #define ARCH_MAP_FLAGS MONO_MMAP_32BIT
69 #define ARCH_MAP_FLAGS 0
72 #define MONO_PROT_RWX (MONO_MMAP_READ|MONO_MMAP_WRITE|MONO_MMAP_EXEC)
74 typedef struct _CodeChunck CodeChunk;
86 unsigned int flags: 8;
87 /* this number of bytes is available to resolve addresses far in memory */
88 unsigned int bsize: 24;
91 struct _MonoCodeManager {
96 #if defined(__native_client_codegen__) && defined(__native_client__)
101 #define ALIGN_INT(val,alignment) (((val) + (alignment - 1)) & ~(alignment - 1))
103 #if defined(__native_client_codegen__) && defined(__native_client__)
104 /* End of text segment, set by linker.
105 * Dynamic text starts on the next allocated page.
108 char *next_dynamic_code_addr = NULL;
111 * This routine gets the next available bundle aligned
112 * pointer in the dynamic code section. It does not check
113 * for the section end, this error will be caught in the
117 allocate_code(intptr_t increment)
120 if (increment < 0) return NULL;
121 increment = increment & kNaClBundleMask ? (increment & ~kNaClBundleMask) + kNaClBundleSize : increment;
122 addr = next_dynamic_code_addr;
123 next_dynamic_code_addr += increment;
128 nacl_is_code_address (void *target)
130 return (char *)target < next_dynamic_code_addr;
133 /* Fill code buffer with arch-specific NOPs. */
135 mono_nacl_fill_code_buffer (guint8 *data, int size);
137 #ifndef USE_JUMP_TABLES
138 const int kMaxPatchDepth = 32;
139 __thread unsigned char **patch_source_base = NULL;
140 __thread unsigned char **patch_dest_base = NULL;
141 __thread int *patch_alloc_size = NULL;
142 __thread int patch_current_depth = -1;
143 __thread int allow_target_modification = 1;
146 nacl_jit_check_init ()
148 if (patch_source_base == NULL) {
149 patch_source_base = g_malloc (kMaxPatchDepth * sizeof(unsigned char *));
150 patch_dest_base = g_malloc (kMaxPatchDepth * sizeof(unsigned char *));
151 patch_alloc_size = g_malloc (kMaxPatchDepth * sizeof(int));
157 nacl_allow_target_modification (int val)
159 #ifndef USE_JUMP_TABLES
160 allow_target_modification = val;
161 #endif /* USE_JUMP_TABLES */
164 /* Given a patch target, modify the target such that patching will work when
165 * the code is copied to the data section.
168 nacl_modify_patch_target (unsigned char *target)
171 * There's no need in patch tricks for jumptables,
172 * as we always patch same jumptable.
174 #ifndef USE_JUMP_TABLES
175 /* This seems like a bit of an ugly way to do this but the advantage
176 * is we don't have to worry about all the conditions in
177 * mono_resolve_patch_target, and it can be used by all the bare uses
183 if (!allow_target_modification) return target;
185 nacl_jit_check_init ();
186 sb = patch_source_base[patch_current_depth];
187 db = patch_dest_base[patch_current_depth];
189 if (target >= sb && (target < sb + patch_alloc_size[patch_current_depth])) {
190 /* Do nothing. target is in the section being generated.
191 * no need to modify, the disp will be the same either way.
194 int target_offset = target - db;
195 target = sb + target_offset;
202 nacl_inverse_modify_patch_target (unsigned char *target)
205 * There's no need in patch tricks for jumptables,
206 * as we always patch same jumptable.
208 #ifndef USE_JUMP_TABLES
213 if (!allow_target_modification) return target;
215 nacl_jit_check_init ();
216 sb = patch_source_base[patch_current_depth];
217 db = patch_dest_base[patch_current_depth];
219 target_offset = target - sb;
220 target = db + target_offset;
226 #endif /* __native_client_codegen && __native_client__ */
228 #define VALLOC_FREELIST_SIZE 16
230 static CRITICAL_SECTION valloc_mutex;
231 static GHashTable *valloc_freelists;
234 codechunk_valloc (guint32 size)
239 if (!valloc_freelists) {
240 InitializeCriticalSection (&valloc_mutex);
241 valloc_freelists = g_hash_table_new (NULL, NULL);
245 * Keep a small freelist of memory blocks to decrease pressure on the kernel memory subsystem to avoid #3321.
247 EnterCriticalSection (&valloc_mutex);
248 freelist = g_hash_table_lookup (valloc_freelists, GUINT_TO_POINTER (size));
250 ptr = freelist->data;
251 memset (ptr, 0, size);
252 freelist = g_slist_remove_link (freelist, freelist);
253 g_hash_table_insert (valloc_freelists, GUINT_TO_POINTER (size), freelist);
255 ptr = mono_valloc (NULL, size + MIN_ALIGN - 1, MONO_PROT_RWX | ARCH_MAP_FLAGS);
257 LeaveCriticalSection (&valloc_mutex);
262 codechunk_vfree (void *ptr, guint32 size)
266 EnterCriticalSection (&valloc_mutex);
267 freelist = g_hash_table_lookup (valloc_freelists, GUINT_TO_POINTER (size));
268 if (!freelist || g_slist_length (freelist) < VALLOC_FREELIST_SIZE) {
269 freelist = g_slist_prepend (freelist, ptr);
270 g_hash_table_insert (valloc_freelists, GUINT_TO_POINTER (size), freelist);
272 mono_vfree (ptr, size);
274 LeaveCriticalSection (&valloc_mutex);
278 codechunk_cleanup (void)
283 if (!valloc_freelists)
285 g_hash_table_iter_init (&iter, valloc_freelists);
286 while (g_hash_table_iter_next (&iter, &key, &value)) {
287 GSList *freelist = value;
290 for (l = freelist; l; l = l->next) {
291 mono_vfree (l->data, GPOINTER_TO_UINT (key));
293 g_slist_free (freelist);
295 g_hash_table_destroy (valloc_freelists);
299 mono_code_manager_init (void)
304 mono_code_manager_cleanup (void)
306 codechunk_cleanup ();
310 * mono_code_manager_new:
312 * Creates a new code manager. A code manager can be used to allocate memory
313 * suitable for storing native code that can be later executed.
314 * A code manager allocates memory from the operating system in large chunks
315 * (typically 64KB in size) so that many methods can be allocated inside them
316 * close together, improving cache locality.
318 * Returns: the new code manager
321 mono_code_manager_new (void)
323 MonoCodeManager *cman = malloc (sizeof (MonoCodeManager));
326 cman->current = NULL;
330 #if defined(__native_client_codegen__) && defined(__native_client__)
331 if (next_dynamic_code_addr == NULL) {
332 const guint kPageMask = 0xFFFF; /* 64K pages */
333 next_dynamic_code_addr = (uintptr_t)(etext + kPageMask) & ~kPageMask;
334 #if defined (__GLIBC__)
335 /* TODO: For now, just jump 64MB ahead to avoid dynamic libraries. */
336 next_dynamic_code_addr += (uintptr_t)0x4000000;
338 /* Workaround bug in service runtime, unable to allocate */
339 /* from the first page in the dynamic code section. */
340 next_dynamic_code_addr += (uintptr_t)0x10000;
343 cman->hash = g_hash_table_new (NULL, NULL);
344 # ifndef USE_JUMP_TABLES
345 if (patch_source_base == NULL) {
346 patch_source_base = g_malloc (kMaxPatchDepth * sizeof(unsigned char *));
347 patch_dest_base = g_malloc (kMaxPatchDepth * sizeof(unsigned char *));
348 patch_alloc_size = g_malloc (kMaxPatchDepth * sizeof(int));
356 * mono_code_manager_new_dynamic:
358 * Creates a new code manager suitable for holding native code that can be
359 * used for single or small methods that need to be deallocated independently
360 * of other native code.
362 * Returns: the new code manager
365 mono_code_manager_new_dynamic (void)
367 MonoCodeManager *cman = mono_code_manager_new ();
374 free_chunklist (CodeChunk *chunk)
378 #if defined(HAVE_VALGRIND_MEMCHECK_H) && defined (VALGRIND_JIT_UNREGISTER_MAP)
379 int valgrind_unregister = 0;
380 if (RUNNING_ON_VALGRIND)
381 valgrind_unregister = 1;
382 #define valgrind_unregister(x) do { if (valgrind_unregister) { VALGRIND_JIT_UNREGISTER_MAP(NULL,x); } } while (0)
384 #define valgrind_unregister(x)
389 mono_profiler_code_chunk_destroy ((gpointer) dead->data);
391 if (dead->flags == CODE_FLAG_MMAP) {
392 codechunk_vfree (dead->data, dead->size);
393 /* valgrind_unregister(dead->data); */
394 } else if (dead->flags == CODE_FLAG_MALLOC) {
397 code_memory_used -= dead->size;
403 * mono_code_manager_destroy:
404 * @cman: a code manager
406 * Free all the memory associated with the code manager @cman.
409 mono_code_manager_destroy (MonoCodeManager *cman)
411 free_chunklist (cman->full);
412 free_chunklist (cman->current);
417 * mono_code_manager_invalidate:
418 * @cman: a code manager
420 * Fill all the memory with an invalid native code value
421 * so that any attempt to execute code allocated in the code
422 * manager @cman will fail. This is used for debugging purposes.
425 mono_code_manager_invalidate (MonoCodeManager *cman)
429 #if defined(__i386__) || defined(__x86_64__)
430 int fill_value = 0xcc; /* x86 break */
432 int fill_value = 0x2a;
435 for (chunk = cman->current; chunk; chunk = chunk->next)
436 memset (chunk->data, fill_value, chunk->size);
437 for (chunk = cman->full; chunk; chunk = chunk->next)
438 memset (chunk->data, fill_value, chunk->size);
442 * mono_code_manager_set_read_only:
443 * @cman: a code manager
445 * Make the code manager read only, so further allocation requests cause an assert.
448 mono_code_manager_set_read_only (MonoCodeManager *cman)
450 cman->read_only = TRUE;
454 * mono_code_manager_foreach:
455 * @cman: a code manager
456 * @func: a callback function pointer
457 * @user_data: additional data to pass to @func
459 * Invokes the callback @func for each different chunk of memory allocated
460 * in the code manager @cman.
463 mono_code_manager_foreach (MonoCodeManager *cman, MonoCodeManagerFunc func, void *user_data)
466 for (chunk = cman->current; chunk; chunk = chunk->next) {
467 if (func (chunk->data, chunk->size, chunk->bsize, user_data))
470 for (chunk = cman->full; chunk; chunk = chunk->next) {
471 if (func (chunk->data, chunk->size, chunk->bsize, user_data))
476 /* BIND_ROOM is the divisor for the chunck of code size dedicated
477 * to binding branches (branches not reachable with the immediate displacement)
478 * bind_size = size/BIND_ROOM;
479 * we should reduce it and make MIN_PAGES bigger for such systems
481 #if defined(__ppc__) || defined(__powerpc__)
489 new_codechunk (int dynamic, int size)
491 int minsize, flags = CODE_FLAG_MMAP;
492 int chunk_size, bsize = 0;
498 flags = CODE_FLAG_MALLOC;
501 pagesize = mono_pagesize ();
505 flags = CODE_FLAG_MALLOC;
507 minsize = pagesize * MIN_PAGES;
509 chunk_size = minsize;
512 chunk_size += pagesize - 1;
513 chunk_size &= ~ (pagesize - 1);
517 bsize = chunk_size / BIND_ROOM;
518 if (bsize < MIN_BSIZE)
520 bsize += MIN_ALIGN -1;
521 bsize &= ~ (MIN_ALIGN - 1);
522 if (chunk_size - size < bsize) {
523 chunk_size = size + bsize;
524 chunk_size += pagesize - 1;
525 chunk_size &= ~ (pagesize - 1);
529 if (flags == CODE_FLAG_MALLOC) {
530 ptr = dlmemalign (MIN_ALIGN, chunk_size + MIN_ALIGN - 1);
534 /* Allocate MIN_ALIGN-1 more than we need so we can still */
535 /* guarantee MIN_ALIGN alignment for individual allocs */
536 /* from mono_code_manager_reserve_align. */
537 ptr = codechunk_valloc (chunk_size);
542 if (flags == CODE_FLAG_MALLOC) {
544 /* Make sure the thunks area is zeroed */
545 memset (ptr, 0, bsize);
549 chunk = malloc (sizeof (CodeChunk));
551 if (flags == CODE_FLAG_MALLOC)
554 mono_vfree (ptr, chunk_size);
558 chunk->size = chunk_size;
560 chunk->flags = flags;
562 chunk->bsize = bsize;
563 mono_profiler_code_chunk_new((gpointer) chunk->data, chunk->size);
565 code_memory_used += chunk_size;
566 mono_runtime_resource_check_limit (MONO_RESOURCE_JIT_CODE, code_memory_used);
567 /*printf ("code chunk at: %p\n", ptr);*/
572 * mono_code_manager_reserve:
573 * @cman: a code manager
574 * @size: size of memory to allocate
575 * @alignment: power of two alignment value
577 * Allocates at least @size bytes of memory inside the code manager @cman.
579 * Returns: the pointer to the allocated memory or #NULL on failure
582 mono_code_manager_reserve_align (MonoCodeManager *cman, int size, int alignment)
584 #if !defined(__native_client__) || !defined(__native_client_codegen__)
585 CodeChunk *chunk, *prev;
587 guint32 align_mask = alignment - 1;
589 g_assert (!cman->read_only);
591 /* eventually allow bigger alignments, but we need to fix the dynamic alloc code to
594 g_assert (alignment <= MIN_ALIGN);
597 ++mono_stats.dynamic_code_alloc_count;
598 mono_stats.dynamic_code_bytes_count += size;
601 if (!cman->current) {
602 cman->current = new_codechunk (cman->dynamic, size);
607 for (chunk = cman->current; chunk; chunk = chunk->next) {
608 if (ALIGN_INT (chunk->pos, alignment) + size <= chunk->size) {
609 chunk->pos = ALIGN_INT (chunk->pos, alignment);
610 /* Align the chunk->data we add to chunk->pos */
611 /* or we can't guarantee proper alignment */
612 ptr = (void*)((((uintptr_t)chunk->data + align_mask) & ~(uintptr_t)align_mask) + chunk->pos);
613 chunk->pos = ((char*)ptr - chunk->data) + size;
618 * no room found, move one filled chunk to cman->full
619 * to keep cman->current from growing too much
622 for (chunk = cman->current; chunk; prev = chunk, chunk = chunk->next) {
623 if (chunk->pos + MIN_ALIGN * 4 <= chunk->size)
626 prev->next = chunk->next;
628 cman->current = chunk->next;
630 chunk->next = cman->full;
634 chunk = new_codechunk (cman->dynamic, size);
637 chunk->next = cman->current;
638 cman->current = chunk;
639 chunk->pos = ALIGN_INT (chunk->pos, alignment);
640 /* Align the chunk->data we add to chunk->pos */
641 /* or we can't guarantee proper alignment */
642 ptr = (void*)((((uintptr_t)chunk->data + align_mask) & ~(uintptr_t)align_mask) + chunk->pos);
643 chunk->pos = ((char*)ptr - chunk->data) + size;
646 unsigned char *temp_ptr, *code_ptr;
647 /* Round up size to next bundle */
648 alignment = kNaClBundleSize;
649 size = (size + kNaClBundleSize) & (~kNaClBundleMask);
650 /* Allocate a temp buffer */
651 temp_ptr = memalign (alignment, size);
652 g_assert (((uintptr_t)temp_ptr & kNaClBundleMask) == 0);
653 /* Allocate code space from the service runtime */
654 code_ptr = allocate_code (size);
655 /* Insert pointer to code space in hash, keyed by buffer ptr */
656 g_hash_table_insert (cman->hash, temp_ptr, code_ptr);
658 #ifndef USE_JUMP_TABLES
659 nacl_jit_check_init ();
661 patch_current_depth++;
662 patch_source_base[patch_current_depth] = temp_ptr;
663 patch_dest_base[patch_current_depth] = code_ptr;
664 patch_alloc_size[patch_current_depth] = size;
665 g_assert (patch_current_depth < kMaxPatchDepth);
673 * mono_code_manager_reserve:
674 * @cman: a code manager
675 * @size: size of memory to allocate
677 * Allocates at least @size bytes of memory inside the code manager @cman.
679 * Returns: the pointer to the allocated memory or #NULL on failure
682 mono_code_manager_reserve (MonoCodeManager *cman, int size)
684 return mono_code_manager_reserve_align (cman, size, MIN_ALIGN);
688 * mono_code_manager_commit:
689 * @cman: a code manager
690 * @data: the pointer returned by mono_code_manager_reserve ()
691 * @size: the size requested in the call to mono_code_manager_reserve ()
692 * @newsize: the new size to reserve
694 * If we reserved too much room for a method and we didn't allocate
695 * already from the code manager, we can get back the excess allocation
696 * for later use in the code manager.
699 mono_code_manager_commit (MonoCodeManager *cman, void *data, int size, int newsize)
701 #if !defined(__native_client__) || !defined(__native_client_codegen__)
702 g_assert (newsize <= size);
704 if (cman->current && (size != newsize) && (data == cman->current->data + cman->current->pos - size)) {
705 cman->current->pos -= size - newsize;
710 g_assert (NACL_BUNDLE_ALIGN_UP(newsize) <= size);
711 code = g_hash_table_lookup (cman->hash, data);
712 g_assert (code != NULL);
713 mono_nacl_fill_code_buffer ((uint8_t*)data + newsize, size - newsize);
714 newsize = NACL_BUNDLE_ALIGN_UP(newsize);
715 g_assert ((GPOINTER_TO_UINT (data) & kNaClBundleMask) == 0);
716 g_assert ((newsize & kNaClBundleMask) == 0);
717 status = nacl_dyncode_create (code, data, newsize);
719 unsigned char *codep;
720 fprintf(stderr, "Error creating Native Client dynamic code section attempted to be\n"
721 "emitted at %p (hex dissasembly of code follows):\n", code);
722 for (codep = data; codep < data + newsize; codep++)
723 fprintf(stderr, "%02x ", *codep);
724 fprintf(stderr, "\n");
725 g_assert_not_reached ();
727 g_hash_table_remove (cman->hash, data);
728 # ifndef USE_JUMP_TABLES
729 g_assert (data == patch_source_base[patch_current_depth]);
730 g_assert (code == patch_dest_base[patch_current_depth]);
731 patch_current_depth--;
732 g_assert (patch_current_depth >= -1);
738 #if defined(__native_client_codegen__) && defined(__native_client__)
740 nacl_code_manager_get_code_dest (MonoCodeManager *cman, void *data)
742 return g_hash_table_lookup (cman->hash, data);
747 * mono_code_manager_size:
748 * @cman: a code manager
749 * @used_size: pointer to an integer for the result
751 * This function can be used to get statistics about a code manager:
752 * the integer pointed to by @used_size will contain how much
753 * memory is actually used inside the code managed @cman.
755 * Returns: the amount of memory allocated in @cman
758 mono_code_manager_size (MonoCodeManager *cman, int *used_size)
763 for (chunk = cman->current; chunk; chunk = chunk->next) {
767 for (chunk = cman->full; chunk; chunk = chunk->next) {
776 #ifdef __native_client_codegen__
777 # if defined(TARGET_ARM)
778 /* Fill empty space with UDF instruction used as halt on ARM. */
780 mono_nacl_fill_code_buffer (guint8 *data, int size)
782 guint32* data32 = (guint32*)data;
784 g_assert(size % 4 == 0);
785 for (i = 0; i < size / 4; i++)
786 data32[i] = 0xE7FEDEFF;
788 # elif (defined(TARGET_X86) || defined(TARGET_AMD64))
789 /* Fill empty space with HLT instruction */
791 mono_nacl_fill_code_buffer(guint8 *data, int size)
793 memset (data, 0xf4, size);