2 * Emit memory access for the front-end.
7 #include <mono/utils/mono-compiler.h>
11 #include <mono/metadata/gc-internals.h>
12 #include <mono/utils/mono-memory-model.h>
16 #include "jit-icalls.h"
18 #define MAX_INLINE_COPIES 10
19 #define MAX_INLINE_COPY_SIZE 10000
22 mini_emit_memset (MonoCompile *cfg, int destreg, int offset, int size, int val, int align)
26 /*FIXME arbitrary hack to avoid unbound code expansion.*/
27 g_assert (size < MAX_INLINE_COPY_SIZE);
31 if ((size <= SIZEOF_REGISTER) && (size <= align)) {
34 MONO_EMIT_NEW_STORE_MEMBASE_IMM (cfg, OP_STOREI1_MEMBASE_IMM, destreg, offset, val);
37 MONO_EMIT_NEW_STORE_MEMBASE_IMM (cfg, OP_STOREI2_MEMBASE_IMM, destreg, offset, val);
40 MONO_EMIT_NEW_STORE_MEMBASE_IMM (cfg, OP_STOREI4_MEMBASE_IMM, destreg, offset, val);
42 #if SIZEOF_REGISTER == 8
44 MONO_EMIT_NEW_STORE_MEMBASE_IMM (cfg, OP_STOREI8_MEMBASE_IMM, destreg, offset, val);
50 val_reg = alloc_preg (cfg);
52 if (SIZEOF_REGISTER == 8)
53 MONO_EMIT_NEW_I8CONST (cfg, val_reg, val);
55 MONO_EMIT_NEW_ICONST (cfg, val_reg, val);
57 if (align < SIZEOF_VOID_P) {
62 if (SIZEOF_VOID_P == 8 && align % 8 == 4)
66 //Unaligned offsets don't naturaly happen in the runtime, so it's ok to be conservative in how we copy
67 //We assume that input src and dest are be aligned to `align` so offset just worsen it
68 int offsets_mask = offset & 0x7; //we only care about the misalignment part
70 if (offsets_mask % 2 == 1)
72 if (offsets_mask % 4 == 2)
74 if (SIZEOF_VOID_P == 8 && offsets_mask % 8 == 4)
78 if (SIZEOF_REGISTER == 8) {
80 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI8_MEMBASE_REG, destreg, offset, val_reg);
88 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, destreg, offset, val_reg);
96 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, destreg, offset, val_reg);
103 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, destreg, offset, val_reg);
110 mini_emit_memcpy (MonoCompile *cfg, int destreg, int doffset, int srcreg, int soffset, int size, int align)
114 /*FIXME arbitrary hack to avoid unbound code expansion.*/
115 g_assert (size < MAX_INLINE_COPY_SIZE);
116 g_assert (align > 0);
118 if (align < SIZEOF_VOID_P) {
126 //Unaligned offsets don't naturaly happen in the runtime, so it's ok to be conservative in how we copy
127 //We assume that input src and dest are be aligned to `align` so offset just worsen it
128 int offsets_mask = (doffset | soffset) & 0x7; //we only care about the misalignment part
130 if (offsets_mask % 2 == 1)
132 if (offsets_mask % 4 == 2)
134 if (SIZEOF_VOID_P == 8 && offsets_mask % 8 == 4)
139 if (SIZEOF_REGISTER == 8) {
141 cur_reg = alloc_preg (cfg);
142 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI8_MEMBASE, cur_reg, srcreg, soffset);
143 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI8_MEMBASE_REG, destreg, doffset, cur_reg);
152 cur_reg = alloc_preg (cfg);
153 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI4_MEMBASE, cur_reg, srcreg, soffset);
154 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, destreg, doffset, cur_reg);
162 cur_reg = alloc_preg (cfg);
163 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI2_MEMBASE, cur_reg, srcreg, soffset);
164 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, destreg, doffset, cur_reg);
172 cur_reg = alloc_preg (cfg);
173 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI1_MEMBASE, cur_reg, srcreg, soffset);
174 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, destreg, doffset, cur_reg);
182 mini_emit_memcpy_internal (MonoCompile *cfg, MonoInst *dest, MonoInst *src, MonoInst *size_ins, int size, int align)
184 /* FIXME: Optimize the case when src/dest is OP_LDADDR */
186 /* We can't do copies at a smaller granule than the provided alignment */
187 if (size_ins || (size / align > MAX_INLINE_COPIES) || !(cfg->opt & MONO_OPT_INTRINS)) {
193 EMIT_NEW_ICONST (cfg, size_ins, size);
194 iargs [2] = size_ins;
195 mono_emit_method_call (cfg, mini_get_memcpy_method (), iargs, NULL);
197 mini_emit_memcpy (cfg, dest->dreg, 0, src->dreg, 0, size, align);
202 mini_emit_memset_internal (MonoCompile *cfg, MonoInst *dest, MonoInst *value_ins, int value, MonoInst *size_ins, int size, int align)
204 /* FIXME: Optimize the case when dest is OP_LDADDR */
206 /* We can't do copies at a smaller granule than the provided alignment */
207 if (value_ins || size_ins || value != 0 || (size / align > MAX_INLINE_COPIES) || !(cfg->opt & MONO_OPT_INTRINS)) {
212 EMIT_NEW_ICONST (cfg, value_ins, value);
213 iargs [1] = value_ins;
216 EMIT_NEW_ICONST (cfg, size_ins, size);
217 iargs [2] = size_ins;
219 mono_emit_method_call (cfg, mini_get_memset_method (), iargs, NULL);
221 mini_emit_memset (cfg, dest->dreg, 0, size, value, align);
226 mini_emit_memcpy_const_size (MonoCompile *cfg, MonoInst *dest, MonoInst *src, int size, int align)
228 mini_emit_memcpy_internal (cfg, dest, src, NULL, size, align);
232 mini_emit_memset_const_size (MonoCompile *cfg, MonoInst *dest, int value, int size, int align)
234 mini_emit_memset_internal (cfg, dest, NULL, value, NULL, size, align);
239 create_write_barrier_bitmap (MonoCompile *cfg, MonoClass *klass, unsigned *wb_bitmap, int offset)
241 MonoClassField *field;
242 gpointer iter = NULL;
244 while ((field = mono_class_get_fields (klass, &iter))) {
247 if (field->type->attrs & FIELD_ATTRIBUTE_STATIC)
249 foffset = klass->valuetype ? field->offset - sizeof (MonoObject): field->offset;
250 if (mini_type_is_reference (mono_field_get_type (field))) {
251 g_assert ((foffset % SIZEOF_VOID_P) == 0);
252 *wb_bitmap |= 1 << ((offset + foffset) / SIZEOF_VOID_P);
254 MonoClass *field_class = mono_class_from_mono_type (field->type);
255 if (field_class->has_references)
256 create_write_barrier_bitmap (cfg, field_class, wb_bitmap, offset + foffset);
262 mini_emit_wb_aware_memcpy (MonoCompile *cfg, MonoClass *klass, MonoInst *iargs[4], int size, int align)
264 int dest_ptr_reg, tmp_reg, destreg, srcreg, offset;
265 unsigned need_wb = 0;
270 /*types with references can't have alignment smaller than sizeof(void*) */
271 if (align < SIZEOF_VOID_P)
274 if (size > 5 * SIZEOF_VOID_P)
277 create_write_barrier_bitmap (cfg, klass, &need_wb, 0);
279 destreg = iargs [0]->dreg;
280 srcreg = iargs [1]->dreg;
283 dest_ptr_reg = alloc_preg (cfg);
284 tmp_reg = alloc_preg (cfg);
287 EMIT_NEW_UNALU (cfg, iargs [0], OP_MOVE, dest_ptr_reg, destreg);
289 while (size >= SIZEOF_VOID_P) {
291 MONO_INST_NEW (cfg, load_inst, OP_LOAD_MEMBASE);
292 load_inst->dreg = tmp_reg;
293 load_inst->inst_basereg = srcreg;
294 load_inst->inst_offset = offset;
295 MONO_ADD_INS (cfg->cbb, load_inst);
297 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREP_MEMBASE_REG, dest_ptr_reg, 0, tmp_reg);
300 mini_emit_write_barrier (cfg, iargs [0], load_inst);
302 offset += SIZEOF_VOID_P;
303 size -= SIZEOF_VOID_P;
306 /*tmp += sizeof (void*)*/
307 if (size >= SIZEOF_VOID_P) {
308 NEW_BIALU_IMM (cfg, iargs [0], OP_PADD_IMM, dest_ptr_reg, dest_ptr_reg, SIZEOF_VOID_P);
309 MONO_ADD_INS (cfg->cbb, iargs [0]);
313 /* Those cannot be references since size < sizeof (void*) */
315 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI4_MEMBASE, tmp_reg, srcreg, offset);
316 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI4_MEMBASE_REG, destreg, offset, tmp_reg);
322 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI2_MEMBASE, tmp_reg, srcreg, offset);
323 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI2_MEMBASE_REG, destreg, offset, tmp_reg);
329 MONO_EMIT_NEW_LOAD_MEMBASE_OP (cfg, OP_LOADI1_MEMBASE, tmp_reg, srcreg, offset);
330 MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, destreg, offset, tmp_reg);
339 mini_emit_memory_copy_internal (MonoCompile *cfg, MonoInst *dest, MonoInst *src, MonoClass *klass, int explicit_align, gboolean native)
344 MonoInst *size_ins = NULL;
345 MonoInst *memcpy_ins = NULL;
349 Fun fact about @native. It's false that @klass will have no ref when @native is true.
350 This happens in pinvoke2. What goes is that marshal.c uses CEE_MONO_LDOBJNATIVE and pass klass.
351 The actual stuff being copied will have no refs, but @klass might.
352 This means we can't assert !(klass->has_references && native).
356 klass = mono_class_from_mono_type (mini_get_underlying_type (&klass->byval_arg));
359 * This check breaks with spilled vars... need to handle it during verification anyway.
360 * g_assert (klass && klass == src->klass && klass == dest->klass);
363 if (mini_is_gsharedvt_klass (klass)) {
365 size_ins = mini_emit_get_gsharedvt_info_klass (cfg, klass, MONO_RGCTX_INFO_VALUE_SIZE);
366 memcpy_ins = mini_emit_get_gsharedvt_info_klass (cfg, klass, MONO_RGCTX_INFO_MEMCPY);
370 size = mono_class_native_size (klass, &align);
372 size = mono_class_value_size (klass, &align);
375 align = SIZEOF_VOID_P;
377 align = explicit_align;
379 if (mini_type_is_reference (&klass->byval_arg)) { // Refs *MUST* be naturally aligned
380 MonoInst *store, *load;
381 int dreg = alloc_ireg_ref (cfg);
383 NEW_LOAD_MEMBASE (cfg, load, OP_LOAD_MEMBASE, dreg, src->dreg, 0);
384 MONO_ADD_INS (cfg->cbb, load);
386 NEW_STORE_MEMBASE (cfg, store, OP_STORE_MEMBASE_REG, dest->dreg, 0, dreg);
387 MONO_ADD_INS (cfg->cbb, store);
389 mini_emit_write_barrier (cfg, dest, src);
390 } else if (cfg->gen_write_barriers && (klass->has_references || size_ins) && !native) { /* if native is true there should be no references in the struct */
391 /* Avoid barriers when storing to the stack */
392 if (!((dest->opcode == OP_ADD_IMM && dest->sreg1 == cfg->frame_reg) ||
393 (dest->opcode == OP_LDADDR))) {
399 context_used = mini_class_check_context_used (cfg, klass);
401 /* It's ok to intrinsify under gsharing since shared code types are layout stable. */
402 if (!size_ins && (cfg->opt & MONO_OPT_INTRINS) && mini_emit_wb_aware_memcpy (cfg, klass, iargs, size, align)) {
403 } else if (size_ins || align < SIZEOF_VOID_P) {
405 iargs [2] = mini_emit_get_rgctx_klass (cfg, context_used, klass, MONO_RGCTX_INFO_KLASS);
407 iargs [2] = mini_emit_runtime_constant (cfg, MONO_PATCH_INFO_CLASS, klass);
408 if (!cfg->compile_aot)
409 mono_class_compute_gc_descriptor (klass);
412 mono_emit_jit_icall (cfg, mono_gsharedvt_value_copy, iargs);
414 mono_emit_jit_icall (cfg, mono_value_copy, iargs);
416 /* We don't unroll more than 5 stores to avoid code bloat. */
417 /*This is harmless and simplify mono_gc_get_range_copy_func */
418 size += (SIZEOF_VOID_P - 1);
419 size &= ~(SIZEOF_VOID_P - 1);
421 EMIT_NEW_ICONST (cfg, iargs [2], size);
422 mono_emit_jit_icall (cfg, mono_gc_get_range_copy_func (), iargs);
431 iargs [2] = size_ins;
432 mini_emit_calli (cfg, mono_method_signature (mini_get_memcpy_method ()), iargs, memcpy_ins, NULL, NULL);
434 mini_emit_memcpy_const_size (cfg, dest, src, size, align);
439 mini_emit_memory_load (MonoCompile *cfg, MonoType *type, MonoInst *src, int offset, int ins_flag)
443 if (ins_flag & MONO_INST_UNALIGNED) {
444 MonoInst *addr, *tmp_var;
446 int size = mono_type_size (type, &align);
449 MonoInst *add_offset;
450 NEW_BIALU_IMM (cfg, add_offset, OP_PADD_IMM, alloc_preg (cfg), src->dreg, offset);
451 MONO_ADD_INS (cfg->cbb, add_offset);
455 tmp_var = mono_compile_create_var (cfg, type, OP_LOCAL);
456 EMIT_NEW_VARLOADA (cfg, addr, tmp_var, tmp_var->inst_vtype);
458 mini_emit_memcpy_const_size (cfg, addr, src, size, 1);
459 EMIT_NEW_TEMPLOAD (cfg, ins, tmp_var->inst_c0);
461 EMIT_NEW_LOAD_MEMBASE_TYPE (cfg, ins, type, src->dreg, offset);
463 ins->flags |= ins_flag;
465 if (ins_flag & MONO_INST_VOLATILE) {
466 /* Volatile loads have acquire semantics, see 12.6.7 in Ecma 335 */
467 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_ACQ);
475 mini_emit_memory_store (MonoCompile *cfg, MonoType *type, MonoInst *dest, MonoInst *value, int ins_flag)
479 if (ins_flag & MONO_INST_VOLATILE) {
480 /* Volatile stores have release semantics, see 12.6.7 in Ecma 335 */
481 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_REL);
484 if (ins_flag & MONO_INST_UNALIGNED) {
485 MonoInst *addr, *mov, *tmp_var;
487 tmp_var = mono_compile_create_var (cfg, type, OP_LOCAL);
488 EMIT_NEW_TEMPSTORE (cfg, mov, tmp_var->inst_c0, value);
489 EMIT_NEW_VARLOADA (cfg, addr, tmp_var, tmp_var->inst_vtype);
490 mini_emit_memory_copy_internal (cfg, dest, addr, mono_class_from_mono_type (type), 1, FALSE);
493 /* FIXME: should check item at sp [1] is compatible with the type of the store. */
495 EMIT_NEW_STORE_MEMBASE_TYPE (cfg, ins, type, dest->dreg, 0, value->dreg);
496 ins->flags |= ins_flag;
497 if (cfg->gen_write_barriers && cfg->method->wrapper_type != MONO_WRAPPER_WRITE_BARRIER &&
498 mini_type_is_reference (type) && !MONO_INS_IS_PCONST_NULL (value)) {
499 /* insert call to write barrier */
500 mini_emit_write_barrier (cfg, dest, value);
505 mini_emit_memory_copy_bytes (MonoCompile *cfg, MonoInst *dest, MonoInst *src, MonoInst *size, int ins_flag)
507 int align = (ins_flag & MONO_INST_UNALIGNED) ? 1 : SIZEOF_VOID_P;
510 * FIXME: It's unclear whether we should be emitting both the acquire
511 * and release barriers for cpblk. It is technically both a load and
512 * store operation, so it seems like that's the sensible thing to do.
514 * FIXME: We emit full barriers on both sides of the operation for
515 * simplicity. We should have a separate atomic memcpy method instead.
517 if (ins_flag & MONO_INST_VOLATILE) {
518 /* Volatile loads have acquire semantics, see 12.6.7 in Ecma 335 */
519 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_SEQ);
522 if ((cfg->opt & MONO_OPT_INTRINS) && (size->opcode == OP_ICONST)) {
523 mini_emit_memcpy_const_size (cfg, dest, src, size->inst_c0, align);
525 mini_emit_memcpy_internal (cfg, dest, src, size, 0, align);
528 if (ins_flag & MONO_INST_VOLATILE) {
529 /* Volatile loads have acquire semantics, see 12.6.7 in Ecma 335 */
530 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_SEQ);
535 mini_emit_memory_init_bytes (MonoCompile *cfg, MonoInst *dest, MonoInst *value, MonoInst *size, int ins_flag)
537 int align = (ins_flag & MONO_INST_UNALIGNED) ? 1 : SIZEOF_VOID_P;
539 if (ins_flag & MONO_INST_VOLATILE) {
540 /* Volatile stores have release semantics, see 12.6.7 in Ecma 335 */
541 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_REL);
544 //FIXME unrolled memset only supports zeroing
545 if ((cfg->opt & MONO_OPT_INTRINS) && (size->opcode == OP_ICONST) && (value->opcode == OP_ICONST) && (value->inst_c0 == 0)) {
546 mini_emit_memset_const_size (cfg, dest, value->inst_c0, size->inst_c0, align);
548 mini_emit_memset_internal (cfg, dest, value, 0, size, 0, align);
554 * If @klass is a valuetype, emit code to copy a value with source address in @src and destination address in @dest.
555 * If @klass is a ref type, copy a pointer instead.
559 mini_emit_memory_copy (MonoCompile *cfg, MonoInst *dest, MonoInst *src, MonoClass *klass, gboolean native, int ins_flag)
561 int explicit_align = 0;
562 if (ins_flag & MONO_INST_UNALIGNED)
566 * FIXME: It's unclear whether we should be emitting both the acquire
567 * and release barriers for cpblk. It is technically both a load and
568 * store operation, so it seems like that's the sensible thing to do.
570 * FIXME: We emit full barriers on both sides of the operation for
571 * simplicity. We should have a separate atomic memcpy method instead.
573 if (ins_flag & MONO_INST_VOLATILE) {
574 /* Volatile loads have acquire semantics, see 12.6.7 in Ecma 335 */
575 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_SEQ);
578 mini_emit_memory_copy_internal (cfg, dest, src, klass, explicit_align, native);
580 if (ins_flag & MONO_INST_VOLATILE) {
581 /* Volatile loads have acquire semantics, see 12.6.7 in Ecma 335 */
582 mini_emit_memory_barrier (cfg, MONO_MEMORY_BARRIER_SEQ);