From a40fe595d0cfe6c65922418de174200ce4873633 Mon Sep 17 00:00:00 2001 From: Stefan Ring Date: Sat, 14 Nov 2009 16:09:19 +0100 Subject: [PATCH] * src/vm/jit/codegen-common.cpp, src/vm/jit/x86_64/codegen.c: Generate MFENCE instructions after (potential) volatile stores, augment patcher info for patching them out again. * src/vm/jit/codegen-common.hpp (codegen_emit_patchable_barrier): New function. * src/vm/jit/patcher-common.cpp (patcher_add_patch_ref): Added return value. * src/vm/jit/patcher-common.hpp: Added disp_mb field to patchref_t. (patcher_add_patch_ref): Added return value. * src/vm/jit/x86_64/codegen.h: Comment. * src/vm/jit/x86_64/patcher.c: Patch out MFENCE for non-volatile writes. --- src/vm/jit/codegen-common.cpp | 28 +++++++++++++++++++++++----- src/vm/jit/codegen-common.hpp | 7 ++++++- src/vm/jit/patcher-common.cpp | 7 ++++++- src/vm/jit/patcher-common.hpp | 9 +++++---- src/vm/jit/x86_64/codegen.c | 26 +++++++++++++++++++++++--- src/vm/jit/x86_64/codegen.h | 3 +++ src/vm/jit/x86_64/patcher.c | 28 ++++++++++++++++++++++++++++ 7 files changed, 94 insertions(+), 14 deletions(-) diff --git a/src/vm/jit/codegen-common.cpp b/src/vm/jit/codegen-common.cpp index 62ee9aecd..9dc00a90b 100644 --- a/src/vm/jit/codegen-common.cpp +++ b/src/vm/jit/codegen-common.cpp @@ -1503,16 +1503,20 @@ bool codegen_emit(jitdata *jd) #if defined(__I386__) // Generate architecture specific instructions. codegen_emit_instruction(jd, iptr); + break; #else + { + fieldinfo* fi; + patchref_t* pr; if (INSTRUCTION_IS_UNRESOLVED(iptr)) { unresolved_field* uf = iptr->sx.s23.s3.uf; fieldtype = uf->fieldref->parseddesc.fd->type; disp = dseg_add_unique_address(cd, 0); - patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp); + pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp); } else { - fieldinfo* fi = iptr->sx.s23.s3.fmiref->p.field; + fi = iptr->sx.s23.s3.fmiref->p.field; fieldtype = fi->type; disp = dseg_add_address(cd, fi->value); @@ -1523,6 +1527,10 @@ bool codegen_emit(jitdata *jd) } } +#if defined(USES_PATCHABLE_MEMORY_BARRIER) + codegen_emit_patchable_barrier(iptr, cd, pr, fi); +#endif + // XXX X86_64: Here We had this: /* This approach is much faster than moving the field address inline into a register. */ @@ -1553,24 +1561,30 @@ bool codegen_emit(jitdata *jd) break; } emit_store_dst(jd, iptr, d); -#endif break; + } +#endif case ICMD_PUTSTATIC: /* ..., value ==> ... */ #if defined(__I386__) // Generate architecture specific instructions. codegen_emit_instruction(jd, iptr); + break; #else + { + fieldinfo* fi; + patchref_t* pr; + if (INSTRUCTION_IS_UNRESOLVED(iptr)) { unresolved_field* uf = iptr->sx.s23.s3.uf; fieldtype = uf->fieldref->parseddesc.fd->type; disp = dseg_add_unique_address(cd, 0); - patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp); + pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp); } else { - fieldinfo* fi = iptr->sx.s23.s3.fmiref->p.field; + fi = iptr->sx.s23.s3.fmiref->p.field; fieldtype = fi->type; disp = dseg_add_address(cd, fi->value); @@ -1610,8 +1624,12 @@ bool codegen_emit(jitdata *jd) M_DST(s1, REG_ITMP1, 0); break; } +#if defined(USES_PATCHABLE_MEMORY_BARRIER) + codegen_emit_patchable_barrier(iptr, cd, pr, fi); #endif break; + } +#endif /* branch operations **********************************************/ diff --git a/src/vm/jit/codegen-common.hpp b/src/vm/jit/codegen-common.hpp index 68fc49bdf..ead70b880 100644 --- a/src/vm/jit/codegen-common.hpp +++ b/src/vm/jit/codegen-common.hpp @@ -1,6 +1,6 @@ /* src/vm/jit/codegen-common.hpp - architecture independent code generator stuff - Copyright (C) 1996-2005, 2006, 2007, 2008 + Copyright (C) 1996-2005, 2006, 2007, 2008, 2009 CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO This file is part of CACAO. @@ -34,6 +34,7 @@ typedef struct branch_label_ref_t branch_label_ref_t; typedef struct jumpref jumpref; typedef struct dataref dataref; +struct patchref_t; #include "config.h" #include "vm/types.h" @@ -228,6 +229,10 @@ void codegen_emit_prolog(jitdata* jd); void codegen_emit_epilog(jitdata* jd); void codegen_emit_instruction(jitdata* jd, instruction* iptr); +#if defined(USES_PATCHABLE_MEMORY_BARRIER) +void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, struct patchref_t *pr, fieldinfo *fi); +#endif + #if defined(ENABLE_INTRP) bool intrp_codegen(jitdata *jd); #endif diff --git a/src/vm/jit/patcher-common.cpp b/src/vm/jit/patcher-common.cpp index 57c6fdb39..dcafac7b7 100644 --- a/src/vm/jit/patcher-common.cpp +++ b/src/vm/jit/patcher-common.cpp @@ -209,9 +209,11 @@ void patcher_list_show(codeinfo *code) Appends a new patcher reference to the list of patching positions. + Returns a pointer to the newly created patchref_t. + *******************************************************************************/ -void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp) +patchref_t *patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp) { codegendata *cd = jd->cd; codeinfo *code = jd->code; @@ -237,6 +239,7 @@ void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp) pr.mpc = patchmpc; pr.datap = 0; pr.disp = disp; + pr.disp_mb = 0; pr.patcher = patcher; pr.ref = ref; pr.mcode = 0; @@ -264,6 +267,8 @@ void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp) cd->lastmcodeptr = cd->mcodeptr + PATCHER_CALL_SIZE; #endif + + return &code->patchers->back(); } diff --git a/src/vm/jit/patcher-common.hpp b/src/vm/jit/patcher-common.hpp index d4a920339..e51582d72 100644 --- a/src/vm/jit/patcher-common.hpp +++ b/src/vm/jit/patcher-common.hpp @@ -48,9 +48,10 @@ typedef struct patchref_t patchref_t; *******************************************************************************/ struct patchref_t { - ptrint mpc; /* absolute position in code segment */ - ptrint datap; /* absolute position in data segment */ - s4 disp; /* displacement of ref in the data segment */ + uintptr_t mpc; /* absolute position in code segment */ + uintptr_t datap; /* absolute position in data segment */ + int32_t disp; /* displacement of ref in the data segment */ + int32_t disp_mb; /* auxiliary code displacement (for membar) */ functionptr patcher; /* patcher function to call */ void* ref; /* reference passed */ uint32_t mcode; /* machine code to be patched back in */ @@ -75,7 +76,7 @@ void patcher_list_free(codeinfo *code); void patcher_list_show(codeinfo *code); #endif -void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp); +patchref_t *patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp); void patcher_resolve(jitdata* jd); diff --git a/src/vm/jit/x86_64/codegen.c b/src/vm/jit/x86_64/codegen.c index 61c86520a..89f12c251 100644 --- a/src/vm/jit/x86_64/codegen.c +++ b/src/vm/jit/x86_64/codegen.c @@ -187,6 +187,22 @@ void codegen_emit_epilog(jitdata* jd) M_RET; } +/** + * Generates a memory barrier to be used after volatile writes. It can be + * patched out later if the field turns out not to be volatile. + */ +void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, patchref_t *pr, fieldinfo *fi) +{ + if (INSTRUCTION_IS_UNRESOLVED(iptr)) { + /* Align on word boundary */ + if ((((intptr_t) cd->mcodeptr) & 3) >= 2) + emit_nop(cd, 4 - (((intptr_t) cd->mcodeptr) & 3)); + /* Displacement for patching out MFENCE */ + pr->disp_mb = (cd->mcodeptr - cd->mcodebase - pr->mpc); + } + if (INSTRUCTION_IS_UNRESOLVED(iptr) || fi->flags & ACC_VOLATILE) + M_MFENCE; +} /** * Generates machine code for one ICMD. @@ -200,6 +216,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) unresolved_method* um; fieldinfo* fi; unresolved_field* uf; + patchref_t* pr; int32_t fieldtype; int32_t s1, s2, s3, d; int32_t disp; @@ -1487,7 +1504,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) /* PROFILE_CYCLE_STOP; */ - patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp); + pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp); /* PROFILE_CYCLE_START; */ } @@ -1527,6 +1544,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) } break; } + codegen_emit_patchable_barrier(iptr, cd, pr, fi); break; case ICMD_GETFIELD: /* ... ==> ..., value */ @@ -1585,7 +1603,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) /* PROFILE_CYCLE_STOP; */ - patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0); + pr = patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0); /* PROFILE_CYCLE_START; */ } @@ -1611,6 +1629,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) M_DST32(s2, s1, disp); break; } + codegen_emit_patchable_barrier(iptr, cd, pr, fi); break; case ICMD_PUTFIELDCONST: /* ..., objectref, value ==> ... */ @@ -1626,7 +1645,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) /* PROFILE_CYCLE_STOP; */ - patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0); + pr = patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0); /* PROFILE_CYCLE_START; */ } @@ -1653,6 +1672,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr) M_LST32(REG_ITMP2, s1, disp); break; } + codegen_emit_patchable_barrier(iptr, cd, pr, fi); break; diff --git a/src/vm/jit/x86_64/codegen.h b/src/vm/jit/x86_64/codegen.h index 4fa215d54..086502be5 100644 --- a/src/vm/jit/x86_64/codegen.h +++ b/src/vm/jit/x86_64/codegen.h @@ -82,6 +82,9 @@ #define BRANCH_UNCONDITIONAL_SIZE 5 /* size in bytes of a branch */ #define BRANCH_CONDITIONAL_SIZE 6 /* size in bytes of a branch */ +/* These NOPs are never executed; they are only used as placeholders during + * code generation. + */ #define BRANCH_NOPS \ do { \ M_NOP; \ diff --git a/src/vm/jit/x86_64/patcher.c b/src/vm/jit/x86_64/patcher.c index 81cc09e0e..be4fd4e7a 100644 --- a/src/vm/jit/x86_64/patcher.c +++ b/src/vm/jit/x86_64/patcher.c @@ -74,6 +74,25 @@ bool patcher_is_valid_trap_instruction_at(void* pc) return (mcode == 0x0b0f); } +/** + * Overwrites the MFENCE instruction at the indicated address with a 3-byte + * NOP. The MFENCE instruction is not allowed to cross a (4-byte) word + * boundary. + * + * @param pc Program counter. + */ +static void patch_out_mfence(void *pc) +{ + uint32_t *p = (uint32_t*) (((uintptr_t) pc) & ~3); + + assert((((uintptr_t) pc) & 3) < 2); + if (((uintptr_t) pc) & 1) + *p = *p & 0x000000ff | 0x001f0f00; + else + *p = *p & 0xff000000 | 0x00001f0f; + + md_icacheflush(p, 4); +} /* patcher_resolve_classref_to_classinfo *************************************** @@ -225,6 +244,9 @@ bool patcher_get_putstatic(patchref_t *pr) // Patch the field value's address. *datap = (uintptr_t) fi->value; + if (pr->disp_mb && !(fi->flags & ACC_VOLATILE)) + patch_out_mfence(ra + pr->disp_mb - 2); + // Synchronize data cache. md_dcacheflush((void*) pr->datap, SIZEOF_VOID_P); @@ -274,6 +296,9 @@ bool patcher_get_putfield(patchref_t *pr) *((int32_t*) (pc + 5)) = fi->offset; } + if (pr->disp_mb && !(fi->flags & ACC_VOLATILE)) + patch_out_mfence(pc + pr->disp_mb - 2); + // Synchronize instruction cache. md_icacheflush(pc, 6 + sizeof(int32_t)); @@ -322,6 +347,9 @@ bool patcher_putfieldconst(patchref_t *pr) *((uint32_t*) (pc + 3)) = fi->offset; } + if (pr->disp_mb && !(fi->flags & ACC_VOLATILE)) + patch_out_mfence(pc + pr->disp_mb - 2); + // Synchronize instruction cache. md_icacheflush(pc, 14 + sizeof(int32_t)); -- 2.25.1