X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Fmetadata%2Fsgen-alloc.c;h=02a1d45bd34dd834bb7c03717fe094fd9ca53bab;hb=2facf4db1f2cc64f9d934ec6ed880765d9ed9074;hp=55cd85bc75b084f63ec8caa3c40f9e1aaf15d814;hpb=33c4f54a02671ded3eeb116d9cd805e8ea697cb7;p=mono.git diff --git a/mono/metadata/sgen-alloc.c b/mono/metadata/sgen-alloc.c index 55cd85bc75b..02a1d45bd34 100644 --- a/mono/metadata/sgen-alloc.c +++ b/mono/metadata/sgen-alloc.c @@ -145,6 +145,34 @@ alloc_degraded (MonoVTable *vtable, size_t size, gboolean for_mature) return p; } +static void +zero_tlab_if_necessary (void *p, size_t size) +{ + if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) { + memset (p, 0, size); + } else { + /* + * This function is called for all allocations in + * TLABs. TLABs originate from fragments, which are + * initialized to be faux arrays. The remainder of + * the fragments are zeroed out at initialization for + * CLEAR_AT_GC, so here we just need to make sure that + * the array header is zeroed. Since we don't know + * whether we're called for the start of a fragment or + * for somewhere in between, we zero in any case, just + * to make sure. + */ + + if (size >= sizeof (MonoArray)) + memset (p, 0, sizeof (MonoArray)); + else { + static guint8 zeros [sizeof (MonoArray)]; + + SGEN_ASSERT (0, !memcmp (p, zeros, size), "TLAB segment must be zeroed out."); + } + } +} + /* * Provide a variant that takes just the vtable for small fixed-size objects. * The aligned size is already computed and stored in vt->gc_descr. @@ -273,9 +301,7 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size) g_assert (0); } - if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) { - memset (p, 0, size); - } + zero_tlab_if_necessary (p, size); } else { size_t alloc_size = 0; if (TLAB_START) @@ -304,9 +330,7 @@ mono_gc_alloc_obj_nolock (MonoVTable *vtable, size_t size) TLAB_REAL_END = TLAB_START + alloc_size; TLAB_TEMP_END = TLAB_START + MIN (SGEN_SCAN_START_SIZE, alloc_size); - if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) { - memset (TLAB_START, 0, alloc_size); - } + zero_tlab_if_necessary (TLAB_START, alloc_size); /* Allocate from the TLAB */ p = (void*)TLAB_NEXT; @@ -347,6 +371,7 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size) TLAB_ACCESS_INIT; size = ALIGN_UP (size); + SGEN_ASSERT (9, size >= sizeof (MonoObject), "Object too small"); g_assert (vtable->gc_descr); if (size > SGEN_MAX_SMALL_OBJ_SIZE) @@ -360,8 +385,7 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size) sgen_set_nursery_scan_start ((char*)p); /*FIXME we should use weak memory ops here. Should help specially on x86. */ - if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) - memset (p, 0, size); + zero_tlab_if_necessary (p, size); } else { int available_in_tlab; char *real_end; @@ -390,8 +414,7 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size) if (!p) return NULL; - if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) - memset (p, 0, size); + zero_tlab_if_necessary (p, size); } else { size_t alloc_size = 0; @@ -407,8 +430,7 @@ mono_gc_try_alloc_obj_nolock (MonoVTable *vtable, size_t size) TLAB_TEMP_END = new_next + MIN (SGEN_SCAN_START_SIZE, alloc_size); sgen_set_nursery_scan_start ((char*)p); - if (nursery_clear_policy == CLEAR_AT_TLAB_CREATION) - memset (new_next, 0, alloc_size); + zero_tlab_if_necessary (new_next, alloc_size); MONO_GC_NURSERY_TLAB_ALLOC ((mword)new_next, alloc_size); } @@ -432,8 +454,30 @@ void* mono_gc_alloc_obj (MonoVTable *vtable, size_t size) { void *res; + + if (!SGEN_CAN_ALIGN_UP (size)) + return NULL; + #ifndef DISABLE_CRITICAL_REGION TLAB_ACCESS_INIT; + + if (G_UNLIKELY (has_per_allocation_action)) { + static int alloc_count; + int current_alloc = InterlockedIncrement (&alloc_count); + + if (verify_before_allocs) { + if ((current_alloc % verify_before_allocs) == 0) + sgen_check_whole_heap_stw (); + } + if (collect_before_allocs) { + if (((current_alloc % collect_before_allocs) == 0) && nursery_section) { + LOCK_GC; + sgen_perform_collection (0, GENERATION_NURSERY, "collect-before-alloc-triggered", TRUE); + UNLOCK_GC; + } + } + } + ENTER_CRITICAL_REGION; res = mono_gc_try_alloc_obj_nolock (vtable, size); if (res) { @@ -454,6 +498,10 @@ void* mono_gc_alloc_vector (MonoVTable *vtable, size_t size, uintptr_t max_length) { MonoArray *arr; + + if (!SGEN_CAN_ALIGN_UP (size)) + return NULL; + #ifndef DISABLE_CRITICAL_REGION TLAB_ACCESS_INIT; ENTER_CRITICAL_REGION; @@ -488,6 +536,9 @@ mono_gc_alloc_array (MonoVTable *vtable, size_t size, uintptr_t max_length, uint MonoArray *arr; MonoArrayBounds *bounds; + if (!SGEN_CAN_ALIGN_UP (size)) + return NULL; + #ifndef DISABLE_CRITICAL_REGION TLAB_ACCESS_INIT; ENTER_CRITICAL_REGION; @@ -526,6 +577,10 @@ void* mono_gc_alloc_string (MonoVTable *vtable, size_t size, gint32 len) { MonoString *str; + + if (!SGEN_CAN_ALIGN_UP (size)) + return NULL; + #ifndef DISABLE_CRITICAL_REGION TLAB_ACCESS_INIT; ENTER_CRITICAL_REGION; @@ -562,7 +617,11 @@ void* mono_gc_alloc_pinned_obj (MonoVTable *vtable, size_t size) { void **p; + + if (!SGEN_CAN_ALIGN_UP (size)) + return NULL; size = ALIGN_UP (size); + LOCK_GC; if (size > SGEN_MAX_SMALL_OBJ_SIZE) { @@ -588,7 +647,12 @@ void* mono_gc_alloc_mature (MonoVTable *vtable) { void **res; - size_t size = ALIGN_UP (vtable->klass->instance_size); + size_t size = vtable->klass->instance_size; + + if (!SGEN_CAN_ALIGN_UP (size)) + return NULL; + size = ALIGN_UP (size); + LOCK_GC; res = alloc_degraded (vtable, size, TRUE); UNLOCK_GC; @@ -685,6 +749,9 @@ create_allocator (int atype) MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset); MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset); + mono_tls_key_set_offset (TLS_KEY_SGEN_TLAB_NEXT_ADDR, tlab_next_addr_offset); + mono_tls_key_set_offset (TLS_KEY_SGEN_TLAB_TEMP_END, tlab_temp_end_offset); + g_assert (tlab_next_addr_offset != -1); g_assert (tlab_temp_end_offset != -1); #endif @@ -692,6 +759,7 @@ create_allocator (int atype) if (!registered) { mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE); mono_register_jit_icall (mono_gc_alloc_vector, "mono_gc_alloc_vector", mono_create_icall_signature ("object ptr int int"), FALSE); + mono_register_jit_icall (mono_gc_alloc_string, "mono_gc_alloc_string", mono_create_icall_signature ("object ptr int int32"), FALSE); registered = TRUE; } @@ -704,19 +772,28 @@ create_allocator (int atype) } else if (atype == ATYPE_VECTOR) { num_params = 2; name = "AllocVector"; + } else if (atype == ATYPE_STRING) { + num_params = 2; + name = "AllocString"; } else { g_assert_not_reached (); } csig = mono_metadata_signature_alloc (mono_defaults.corlib, num_params); - csig->ret = &mono_defaults.object_class->byval_arg; - for (i = 0; i < num_params; ++i) - csig->params [i] = &mono_defaults.int_class->byval_arg; + if (atype == ATYPE_STRING) { + csig->ret = &mono_defaults.string_class->byval_arg; + csig->params [0] = &mono_defaults.int_class->byval_arg; + csig->params [1] = &mono_defaults.int32_class->byval_arg; + } else { + csig->ret = &mono_defaults.object_class->byval_arg; + for (i = 0; i < num_params; ++i) + csig->params [i] = &mono_defaults.int_class->byval_arg; + } mb = mono_mb_new (mono_defaults.object_class, name, MONO_WRAPPER_ALLOC); #ifndef DISABLE_JIT - size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg); + size_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg); if (atype == ATYPE_NORMAL || atype == ATYPE_SMALL) { /* size = vtable->klass->instance_size; */ mono_mb_emit_ldarg (mb, 0); @@ -727,18 +804,35 @@ create_allocator (int atype) mono_mb_emit_byte (mb, CEE_ADD); /* FIXME: assert instance_size stays a 4 byte integer */ mono_mb_emit_byte (mb, CEE_LDIND_U4); + mono_mb_emit_byte (mb, CEE_CONV_I); mono_mb_emit_stloc (mb, size_var); } else if (atype == ATYPE_VECTOR) { MonoExceptionClause *clause; - int pos, pos_leave; + int pos, pos_leave, pos_error; MonoClass *oom_exc_class; MonoMethod *ctor; - /* n > MONO_ARRAY_MAX_INDEX -> OverflowException */ + /* + * n > MONO_ARRAY_MAX_INDEX => OutOfMemoryException + * n < 0 => OverflowException + * + * We can do an unsigned comparison to catch both cases, then in the error + * case compare signed to distinguish between them. + */ mono_mb_emit_ldarg (mb, 1); mono_mb_emit_icon (mb, MONO_ARRAY_MAX_INDEX); + mono_mb_emit_byte (mb, CEE_CONV_U); pos = mono_mb_emit_short_branch (mb, CEE_BLE_UN_S); + + mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX); + mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN); + mono_mb_emit_ldarg (mb, 1); + mono_mb_emit_icon (mb, 0); + pos_error = mono_mb_emit_short_branch (mb, CEE_BLT_S); + mono_mb_emit_exception (mb, "OutOfMemoryException", NULL); + mono_mb_patch_short_branch (mb, pos_error); mono_mb_emit_exception (mb, "OverflowException", NULL); + mono_mb_patch_short_branch (mb, pos); clause = mono_image_alloc0 (mono_defaults.corlib, sizeof (MonoExceptionClause)); @@ -752,6 +846,7 @@ create_allocator (int atype) mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, sizes.element_size)); mono_mb_emit_byte (mb, CEE_ADD); mono_mb_emit_byte (mb, CEE_LDIND_U4); + mono_mb_emit_byte (mb, CEE_CONV_I); /* * n */ mono_mb_emit_ldarg (mb, 1); @@ -785,6 +880,39 @@ create_allocator (int atype) mono_mb_set_clauses (mb, 1, clause); mono_mb_patch_branch (mb, pos_leave); /* end catch */ + } else if (atype == ATYPE_STRING) { + int pos; + + /* + * a string allocator method takes the args: (vtable, len) + * + * bytes = sizeof (MonoString) + ((len + 1) * 2) + * + * condition: + * + * bytes <= INT32_MAX - (SGEN_ALLOC_ALIGN - 1) + * + * therefore: + * + * sizeof (MonoString) + ((len + 1) * 2) <= INT32_MAX - (SGEN_ALLOC_ALIGN - 1) + * len <= (INT32_MAX - (SGEN_ALLOC_ALIGN - 1) - sizeof (MonoString)) / 2 - 1 + */ + mono_mb_emit_ldarg (mb, 1); + mono_mb_emit_icon (mb, (INT32_MAX - (SGEN_ALLOC_ALIGN - 1) - sizeof (MonoString)) / 2 - 1); + pos = mono_mb_emit_short_branch (mb, MONO_CEE_BLE_UN_S); + + mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX); + mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN); + mono_mb_emit_exception (mb, "OutOfMemoryException", NULL); + mono_mb_patch_short_branch (mb, pos); + + mono_mb_emit_ldarg (mb, 1); + mono_mb_emit_icon (mb, 1); + mono_mb_emit_byte (mb, MONO_CEE_SHL); + //WE manually fold the above + 2 here + mono_mb_emit_icon (mb, sizeof (MonoString) + 2); + mono_mb_emit_byte (mb, CEE_ADD); + mono_mb_emit_stloc (mb, size_var); } else { g_assert_not_reached (); } @@ -812,7 +940,7 @@ create_allocator (int atype) /* tlab_next_addr (local) = tlab_next_addr (TLS var) */ tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg); - EMIT_TLS_ACCESS (mb, tlab_next_addr, tlab_next_addr_offset); + EMIT_TLS_ACCESS (mb, tlab_next_addr, TLS_KEY_SGEN_TLAB_NEXT_ADDR); mono_mb_emit_stloc (mb, tlab_next_addr_var); /* p = (void**)tlab_next; */ @@ -831,7 +959,7 @@ create_allocator (int atype) /* if (G_LIKELY (new_next < tlab_temp_end)) */ mono_mb_emit_ldloc (mb, new_next_var); - EMIT_TLS_ACCESS (mb, tlab_temp_end, tlab_temp_end_offset); + EMIT_TLS_ACCESS (mb, tlab_temp_end, TLS_KEY_SGEN_TLAB_TEMP_END); slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S); /* Slowpath */ @@ -849,6 +977,9 @@ create_allocator (int atype) } else if (atype == ATYPE_VECTOR) { mono_mb_emit_ldarg (mb, 1); mono_mb_emit_icall (mb, mono_gc_alloc_vector); + } else if (atype == ATYPE_STRING) { + mono_mb_emit_ldarg (mb, 1); + mono_mb_emit_icall (mb, mono_gc_alloc_string); } else { g_assert_not_reached (); } @@ -883,6 +1014,22 @@ create_allocator (int atype) #else mono_mb_emit_byte (mb, CEE_STIND_I4); #endif + } else if (atype == ATYPE_STRING) { + /* need to set length and clear the last char */ + /* s->length = len; */ + mono_mb_emit_ldloc (mb, p_var); + mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoString, length)); + mono_mb_emit_byte (mb, MONO_CEE_ADD); + mono_mb_emit_ldarg (mb, 1); + mono_mb_emit_byte (mb, MONO_CEE_STIND_I4); + /* s->chars [len] = 0; */ + mono_mb_emit_ldloc (mb, p_var); + mono_mb_emit_ldloc (mb, size_var); + mono_mb_emit_icon (mb, 2); + mono_mb_emit_byte (mb, MONO_CEE_SUB); + mono_mb_emit_byte (mb, MONO_CEE_ADD); + mono_mb_emit_icon (mb, 0); + mono_mb_emit_byte (mb, MONO_CEE_STIND_I2); } /* @@ -915,10 +1062,9 @@ create_allocator (int atype) * object allocate (MonoVTable *vtable) */ MonoMethod* -mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box) +mono_gc_get_managed_allocator (MonoClass *klass, gboolean for_box) { #ifdef MANAGED_ALLOCATION - MonoClass *klass = vtable->klass; #ifdef HAVE_KW_THREAD int tlab_next_offset = -1; @@ -929,21 +1075,21 @@ mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box) if (tlab_next_offset == -1 || tlab_temp_end_offset == -1) return NULL; #endif - + if (collect_before_allocs) + return NULL; if (!mono_runtime_has_tls_get ()) return NULL; if (klass->instance_size > tlab_size) return NULL; + if (klass->has_finalize || mono_class_is_marshalbyref (klass) || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS)) return NULL; if (klass->rank) return NULL; if (klass->byval_arg.type == MONO_TYPE_STRING) - return NULL; - if (collect_before_allocs) - return NULL; - - if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE) + return mono_gc_get_managed_allocator_by_type (ATYPE_STRING); + /* Generic classes have dynamic field and can go above MAX_SMALL_OBJ_SIZE. */ + if (ALIGN_TO (klass->instance_size, ALLOC_ALIGN) < MAX_SMALL_OBJ_SIZE && !mono_class_is_open_constructed_type (&klass->byval_arg)) return mono_gc_get_managed_allocator_by_type (ATYPE_SMALL); else return mono_gc_get_managed_allocator_by_type (ATYPE_NORMAL); @@ -953,11 +1099,9 @@ mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box) } MonoMethod* -mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank) +mono_gc_get_managed_array_allocator (MonoClass *klass) { #ifdef MANAGED_ALLOCATION - MonoClass *klass = vtable->klass; - #ifdef HAVE_KW_THREAD int tlab_next_offset = -1; int tlab_temp_end_offset = -1; @@ -968,7 +1112,7 @@ mono_gc_get_managed_array_allocator (MonoVTable *vtable, int rank) return NULL; #endif - if (rank != 1) + if (klass->rank != 1) return NULL; if (!mono_runtime_has_tls_get ()) return NULL; @@ -1002,11 +1146,21 @@ mono_gc_get_managed_allocator_by_type (int atype) if (!mono_runtime_has_tls_get ()) return NULL; - mono_loader_lock (); res = alloc_method_cache [atype]; - if (!res) - res = alloc_method_cache [atype] = create_allocator (atype); - mono_loader_unlock (); + if (res) + return res; + + res = create_allocator (atype); + LOCK_GC; + if (alloc_method_cache [atype]) { + mono_free_method (res); + res = alloc_method_cache [atype]; + } else { + mono_memory_barrier (); + alloc_method_cache [atype] = res; + } + UNLOCK_GC; + return res; #else return NULL;