+enum {
+ ATYPE_NORMAL,
+ ATYPE_NUM
+};
+
+/* FIXME: Do this in the JIT, where specialized allocation sequences can be created
+ * for each class. This is currently not easy to do, as it is hard to generate basic
+ * blocks + branches, but it is easy with the linear IL codebase.
+ */
+static MonoMethod*
+create_allocator (int atype)
+{
+ int tlab_next_addr_offset = -1;
+ int tlab_temp_end_offset = -1;
+ int p_var, size_var, tlab_next_addr_var, new_next_var;
+ guint32 slowpath_branch;
+ MonoMethodBuilder *mb;
+ MonoMethod *res;
+ MonoMethodSignature *csig;
+ static gboolean registered = FALSE;
+
+ MONO_THREAD_VAR_OFFSET (tlab_next_addr, tlab_next_addr_offset);
+ MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
+
+ g_assert (tlab_next_addr_offset != -1);
+ g_assert (tlab_temp_end_offset != -1);
+
+ g_assert (atype == ATYPE_NORMAL);
+
+ if (!registered) {
+ mono_register_jit_icall (mono_gc_alloc_obj, "mono_gc_alloc_obj", mono_create_icall_signature ("object ptr int"), FALSE);
+ registered = TRUE;
+ }
+
+ csig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
+ csig->ret = &mono_defaults.object_class->byval_arg;
+ csig->params [0] = &mono_defaults.int_class->byval_arg;
+
+ mb = mono_mb_new (mono_defaults.object_class, "Alloc", MONO_WRAPPER_ALLOC);
+ size_var = mono_mb_add_local (mb, &mono_defaults.int32_class->byval_arg);
+ /* size = vtable->klass->instance_size; */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoVTable, klass));
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_icon (mb, G_STRUCT_OFFSET (MonoClass, instance_size));
+ mono_mb_emit_byte (mb, CEE_ADD);
+ /* FIXME: assert instance_size stays a 4 byte integer */
+ mono_mb_emit_byte (mb, CEE_LDIND_U4);
+ mono_mb_emit_stloc (mb, size_var);
+
+ /* size += ALLOC_ALIGN - 1; */
+ mono_mb_emit_ldloc (mb, size_var);
+ mono_mb_emit_icon (mb, ALLOC_ALIGN - 1);
+ mono_mb_emit_byte (mb, CEE_ADD);
+ /* size &= ~(ALLOC_ALIGN - 1); */
+ mono_mb_emit_icon (mb, ~(ALLOC_ALIGN - 1));
+ mono_mb_emit_byte (mb, CEE_AND);
+ mono_mb_emit_stloc (mb, size_var);
+
+ /*
+ * We need to modify tlab_next, but the JIT only supports reading, so we read
+ * another tls var holding its address instead.
+ */
+
+ /* tlab_next_addr (local) = tlab_next_addr (TLS var) */
+ tlab_next_addr_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+ mono_mb_emit_byte (mb, CEE_MONO_TLS);
+ mono_mb_emit_i4 (mb, tlab_next_addr_offset);
+ mono_mb_emit_stloc (mb, tlab_next_addr_var);
+
+ /* p = (void**)tlab_next; */
+ p_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldloc (mb, tlab_next_addr_var);
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_stloc (mb, p_var);
+
+ /* new_next = (char*)p + size; */
+ new_next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldloc (mb, p_var);
+ mono_mb_emit_ldloc (mb, size_var);
+ mono_mb_emit_byte (mb, CEE_CONV_I);
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_stloc (mb, new_next_var);
+
+ /* tlab_next = new_next */
+ mono_mb_emit_ldloc (mb, tlab_next_addr_var);
+ mono_mb_emit_ldloc (mb, new_next_var);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ /* if (G_LIKELY (new_next < tlab_temp_end)) */
+ mono_mb_emit_ldloc (mb, new_next_var);
+ mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+ mono_mb_emit_byte (mb, CEE_MONO_TLS);
+ mono_mb_emit_i4 (mb, tlab_temp_end_offset);
+ slowpath_branch = mono_mb_emit_short_branch (mb, MONO_CEE_BLT_UN_S);
+
+ /* Slowpath */
+
+ mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+ mono_mb_emit_byte (mb, CEE_MONO_NOT_TAKEN);
+
+ /* FIXME: mono_gc_alloc_obj takes a 'size_t' as an argument, not an int32 */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldloc (mb, size_var);
+ mono_mb_emit_icall (mb, mono_gc_alloc_obj);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ /* Fastpath */
+ mono_mb_patch_short_branch (mb, slowpath_branch);
+
+ /* FIXME: Memory barrier */
+
+ /* *p = vtable; */
+ mono_mb_emit_ldloc (mb, p_var);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ /* return p */
+ mono_mb_emit_ldloc (mb, p_var);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ res = mono_mb_create_method (mb, csig, 8);
+ mono_mb_free (mb);
+ mono_method_get_header (res)->init_locals = FALSE;
+ return res;
+}
+
+static MonoMethod* alloc_method_cache [ATYPE_NUM];
+
+/*
+ * Generate an allocator method implementing the fast path of mono_gc_alloc_obj ().
+ * The signature of the called method is:
+ * object allocate (MonoVTable *vtable)
+ */
+MonoMethod*
+mono_gc_get_managed_allocator (MonoVTable *vtable, gboolean for_box)
+{
+ int tlab_next_offset = -1;
+ int tlab_temp_end_offset = -1;
+ MonoClass *klass = vtable->klass;
+ MONO_THREAD_VAR_OFFSET (tlab_next, tlab_next_offset);
+ MONO_THREAD_VAR_OFFSET (tlab_temp_end, tlab_temp_end_offset);
+
+ if (tlab_next_offset == -1 || tlab_temp_end_offset == -1)
+ return NULL;
+ if (klass->instance_size > tlab_size)
+ return NULL;
+ if (klass->has_finalize || klass->marshalbyref || (mono_profiler_get_events () & MONO_PROFILE_ALLOCATIONS))
+ return NULL;
+ if (klass->rank)
+ return NULL;
+ if (klass->byval_arg.type == MONO_TYPE_STRING)
+ return NULL;
+ if (collect_before_allocs)
+ return NULL;
+
+ return mono_gc_get_managed_allocator_by_type (0);
+}
+
+int
+mono_gc_get_managed_allocator_type (MonoMethod *managed_alloc)
+{
+ return 0;
+}
+
+MonoMethod*
+mono_gc_get_managed_allocator_by_type (int atype)
+{
+ MonoMethod *res;
+
+ mono_loader_lock ();
+ res = alloc_method_cache [atype];
+ if (!res)
+ res = alloc_method_cache [atype] = create_allocator (atype);
+ mono_loader_unlock ();
+ return res;
+}
+
+guint32
+mono_gc_get_managed_allocator_types (void)
+{
+ return ATYPE_NUM;
+}
+
+static MonoMethod *write_barrier_method;
+
+MonoMethod*
+mono_gc_get_write_barrier (void)
+{
+ MonoMethod *res;
+ int remset_offset = -1;
+ int remset_var, next_var;
+ MonoMethodBuilder *mb;
+ MonoMethodSignature *sig;
+ int label1, label2;
+
+ MONO_THREAD_VAR_OFFSET (remembered_set, remset_offset);
+
+ // FIXME: Maybe create a separate version for ctors (the branch would be
+ // correctly predicted more times)
+ if (write_barrier_method)
+ return write_barrier_method;
+
+ /* Create the IL version of mono_gc_barrier_generic_store () */
+ sig = mono_metadata_signature_alloc (mono_defaults.corlib, 2);
+ sig->ret = &mono_defaults.void_class->byval_arg;
+ sig->params [0] = &mono_defaults.int_class->byval_arg;
+ sig->params [1] = &mono_defaults.object_class->byval_arg;
+
+ mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
+
+ /* ptr_in_nursery () check */
+#ifdef ALIGN_NURSERY
+ /*
+ * Masking out the bits might be faster, but we would have to use 64 bit
+ * immediates, which might be slower.
+ */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
+ mono_mb_emit_byte (mb, CEE_SHR_UN);
+ mono_mb_emit_icon (mb, (mword)nursery_start >> DEFAULT_NURSERY_BITS);
+ label1 = mono_mb_emit_branch (mb, CEE_BNE_UN);
+#else
+ // FIXME:
+ g_assert_not_reached ();
+#endif
+
+ /* Don't need write barrier case */
+ /* do the assignment */
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldarg (mb, 1);
+ /* Don't use STIND_REF, as it would cause infinite recursion */
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ /* Need write barrier case */
+ mono_mb_patch_branch (mb, label1);
+
+ if (remset_offset == -1)
+ // FIXME:
+ g_assert_not_reached ();
+
+ // remset_var = remembered_set;
+ remset_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_byte (mb, MONO_CUSTOM_PREFIX);
+ mono_mb_emit_byte (mb, CEE_MONO_TLS);
+ mono_mb_emit_i4 (mb, remset_offset);
+ mono_mb_emit_stloc (mb, remset_var);
+
+ // next_var = rs->store_next
+ next_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
+ mono_mb_emit_ldloc (mb, remset_var);
+ mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, store_next));
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ mono_mb_emit_stloc (mb, next_var);
+
+ // if (rs->store_next < rs->end_set) {
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_ldloc (mb, remset_var);
+ mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, end_set));
+ mono_mb_emit_byte (mb, CEE_LDIND_I);
+ label2 = mono_mb_emit_branch (mb, CEE_BGE);
+
+ /* write barrier fast path */
+ // *(rs->store_next++) = (mword)ptr;
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_icon (mb, sizeof (gpointer));
+ mono_mb_emit_byte (mb, CEE_ADD);
+ mono_mb_emit_stloc (mb, next_var);
+
+ mono_mb_emit_ldloc (mb, remset_var);
+ mono_mb_emit_ldflda (mb, G_STRUCT_OFFSET (RememberedSet, store_next));
+ mono_mb_emit_ldloc (mb, next_var);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+
+ // *(void**)ptr = value;
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldarg (mb, 1);
+ mono_mb_emit_byte (mb, CEE_STIND_I);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ /* write barrier slow path */
+ mono_mb_patch_branch (mb, label2);
+
+ mono_mb_emit_ldarg (mb, 0);
+ mono_mb_emit_ldarg (mb, 1);
+ mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_store);
+ mono_mb_emit_byte (mb, CEE_RET);
+
+ res = mono_mb_create_method (mb, sig, 16);
+ mono_mb_free (mb);
+
+ mono_loader_lock ();
+ if (write_barrier_method) {
+ /* Already created */
+ mono_free_method (res);
+ } else {
+ /* double-checked locking */
+ mono_memory_barrier ();
+ write_barrier_method = res;
+ }
+ mono_loader_unlock ();
+
+ return write_barrier_method;
+}
+