* src/vm/jit/codegen-common.cpp, src/vm/jit/x86_64/codegen.c: Generate

MFENCE instructions after (potential) volatile stores, augment patcher info
for patching them out again.
* src/vm/jit/codegen-common.hpp (codegen_emit_patchable_barrier): New function.
* src/vm/jit/patcher-common.cpp (patcher_add_patch_ref): Added return value.
* src/vm/jit/patcher-common.hpp: Added disp_mb field to patchref_t.
(patcher_add_patch_ref): Added return value.
* src/vm/jit/x86_64/codegen.h: Comment.
* src/vm/jit/x86_64/patcher.c: Patch out MFENCE for non-volatile writes.

diff --git a/src/vm/jit/codegen-common.cpp b/src/vm/jit/codegen-common.cpp
index 62ee9aecdda1f8cf582f5c8535bd9e4c2f00ab78..9dc00a90bc10f91ff7d1d637f16b88be837bd65c 100644 (file)
--- a/src/vm/jit/codegen-common.cpp
+++ b/src/vm/jit/codegen-common.cpp
@@ -1503,16 +1503,20 @@ bool codegen_emit(jitdata *jd)
 #if defined(__I386__)
                                // Generate architecture specific instructions.
                                codegen_emit_instruction(jd, iptr);
+                               break;
 #else
+                       {
+                               fieldinfo* fi;
+                               patchref_t* pr;
                                if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
                                        unresolved_field* uf = iptr->sx.s23.s3.uf;
                                        fieldtype = uf->fieldref->parseddesc.fd->type;
                                        disp      = dseg_add_unique_address(cd, 0);
 
-                                       patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
+                                       pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
                                }
                                else {
-                                       fieldinfo* fi = iptr->sx.s23.s3.fmiref->p.field;
+                                       fi        = iptr->sx.s23.s3.fmiref->p.field;
                                        fieldtype = fi->type;
                                        disp      = dseg_add_address(cd, fi->value);
 
@@ -1523,6 +1527,10 @@ bool codegen_emit(jitdata *jd)
                                        }
                                }
 
+#if defined(USES_PATCHABLE_MEMORY_BARRIER)
+                               codegen_emit_patchable_barrier(iptr, cd, pr, fi);
+#endif
+
                                // XXX X86_64: Here We had this:
                                /* This approach is much faster than moving the field
                                   address inline into a register. */
@@ -1553,24 +1561,30 @@ bool codegen_emit(jitdata *jd)
                                        break;
                                }
                                emit_store_dst(jd, iptr, d);
-#endif
                                break;
+                       }
+#endif
 
                        case ICMD_PUTSTATIC:  /* ..., value  ==> ...                      */
 
 #if defined(__I386__)
                                // Generate architecture specific instructions.
                                codegen_emit_instruction(jd, iptr);
+                               break;
 #else
+                       {
+                               fieldinfo* fi;
+                               patchref_t* pr;
+
                                if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
                                        unresolved_field* uf = iptr->sx.s23.s3.uf;
                                        fieldtype = uf->fieldref->parseddesc.fd->type;
                                        disp      = dseg_add_unique_address(cd, 0);
 
-                                       patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
+                                       pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
                                }
                                else {
-                                       fieldinfo* fi = iptr->sx.s23.s3.fmiref->p.field;
+                                       fi = iptr->sx.s23.s3.fmiref->p.field;
                                        fieldtype = fi->type;
                                        disp      = dseg_add_address(cd, fi->value);
 
@@ -1610,8 +1624,12 @@ bool codegen_emit(jitdata *jd)
                                        M_DST(s1, REG_ITMP1, 0);
                                        break;
                                }
+#if defined(USES_PATCHABLE_MEMORY_BARRIER)
+                               codegen_emit_patchable_barrier(iptr, cd, pr, fi);
 #endif
                                break;
+                       }
+#endif
 
                        /* branch operations **********************************************/
 

diff --git a/src/vm/jit/codegen-common.hpp b/src/vm/jit/codegen-common.hpp
index 68fc49bdff27c263109524fe155e1f75d8680ccb..ead70b880c053c91f3f0a73de8c17ed730d29bc2 100644 (file)
--- a/src/vm/jit/codegen-common.hpp
+++ b/src/vm/jit/codegen-common.hpp
@@ -1,6 +1,6 @@
 /* src/vm/jit/codegen-common.hpp - architecture independent code generator stuff
 
-   Copyright (C) 1996-2005, 2006, 2007, 2008
+   Copyright (C) 1996-2005, 2006, 2007, 2008, 2009
    CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
 
    This file is part of CACAO.
@@ -34,6 +34,7 @@ typedef struct branch_label_ref_t     branch_label_ref_t;
 typedef struct jumpref                jumpref;
 typedef struct dataref                dataref;
 
+struct patchref_t;
 
 #include "config.h"
 #include "vm/types.h"
@@ -228,6 +229,10 @@ void codegen_emit_prolog(jitdata* jd);
 void codegen_emit_epilog(jitdata* jd);
 void codegen_emit_instruction(jitdata* jd, instruction* iptr);
 
+#if defined(USES_PATCHABLE_MEMORY_BARRIER)
+void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, struct patchref_t *pr, fieldinfo *fi);
+#endif
+
 #if defined(ENABLE_INTRP)
 bool intrp_codegen(jitdata *jd);
 #endif

diff --git a/src/vm/jit/patcher-common.cpp b/src/vm/jit/patcher-common.cpp
index 57c6fdb39005537ba738f0c2e160324d418083d1..dcafac7b7eedad04c3333d23124af04a9e970710 100644 (file)
--- a/src/vm/jit/patcher-common.cpp
+++ b/src/vm/jit/patcher-common.cpp
@@ -209,9 +209,11 @@ void patcher_list_show(codeinfo *code)
 
    Appends a new patcher reference to the list of patching positions.
 
+   Returns a pointer to the newly created patchref_t.
+
 *******************************************************************************/
 
-void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp)
+patchref_t *patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp)
 {
        codegendata *cd   = jd->cd;
        codeinfo    *code = jd->code;
@@ -237,6 +239,7 @@ void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp)
        pr.mpc     = patchmpc;
        pr.datap   = 0;
        pr.disp    = disp;
+       pr.disp_mb = 0;
        pr.patcher = patcher;
        pr.ref     = ref;
        pr.mcode   = 0;
@@ -264,6 +267,8 @@ void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp)
 
        cd->lastmcodeptr = cd->mcodeptr + PATCHER_CALL_SIZE;
 #endif
+
+       return &code->patchers->back();
 }
 
 

diff --git a/src/vm/jit/patcher-common.hpp b/src/vm/jit/patcher-common.hpp
index d4a920339111437721b1703e801d809fc5f3d8ec..e51582d72eda21f94793df501417af814b576151 100644 (file)
--- a/src/vm/jit/patcher-common.hpp
+++ b/src/vm/jit/patcher-common.hpp
@@ -48,9 +48,10 @@ typedef struct patchref_t patchref_t;
 *******************************************************************************/
 
 struct patchref_t {
-       ptrint       mpc;           /* absolute position in code segment          */
-       ptrint       datap;         /* absolute position in data segment          */
-       s4           disp;          /* displacement of ref in the data segment    */
+       uintptr_t    mpc;           /* absolute position in code segment          */
+       uintptr_t    datap;         /* absolute position in data segment          */
+       int32_t      disp;          /* displacement of ref in the data segment    */
+       int32_t      disp_mb;       /* auxiliary code displacement (for membar)   */
        functionptr  patcher;       /* patcher function to call                   */
        void*        ref;           /* reference passed                           */
        uint32_t     mcode;         /* machine code to be patched back in         */
@@ -75,7 +76,7 @@ void patcher_list_free(codeinfo *code);
 void patcher_list_show(codeinfo *code);
 #endif
 
-void patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp);
+patchref_t *patcher_add_patch_ref(jitdata *jd, functionptr patcher, void* ref, s4 disp);
 
 void patcher_resolve(jitdata* jd);
 

diff --git a/src/vm/jit/x86_64/codegen.c b/src/vm/jit/x86_64/codegen.c
index 61c86520adac4c06c365ecc0f722c52b250af9f2..89f12c251cdbaa9183b87bcd0ba28715238a6876 100644 (file)
--- a/src/vm/jit/x86_64/codegen.c
+++ b/src/vm/jit/x86_64/codegen.c
@@ -187,6 +187,22 @@ void codegen_emit_epilog(jitdata* jd)
        M_RET;
 }
 
+/**
+ * Generates a memory barrier to be used after volatile writes. It can be
+ * patched out later if the field turns out not to be volatile.
+ */
+void codegen_emit_patchable_barrier(instruction *iptr, codegendata *cd, patchref_t *pr, fieldinfo *fi)
+{
+       if (INSTRUCTION_IS_UNRESOLVED(iptr)) {
+               /* Align on word boundary */
+               if ((((intptr_t) cd->mcodeptr) & 3) >= 2)
+                       emit_nop(cd, 4 - (((intptr_t) cd->mcodeptr) & 3));
+               /* Displacement for patching out MFENCE */
+               pr->disp_mb = (cd->mcodeptr - cd->mcodebase - pr->mpc);
+       }
+       if (INSTRUCTION_IS_UNRESOLVED(iptr) || fi->flags & ACC_VOLATILE)
+               M_MFENCE;
+}
 
 /**
  * Generates machine code for one ICMD.
@@ -200,6 +216,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
        unresolved_method*  um;
        fieldinfo*          fi;
        unresolved_field*   uf;
+       patchref_t*         pr;
        int32_t             fieldtype;
        int32_t             s1, s2, s3, d;
        int32_t             disp;
@@ -1487,7 +1504,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 
 /*                             PROFILE_CYCLE_STOP; */
 
-                               patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
+                               pr = patcher_add_patch_ref(jd, PATCHER_get_putstatic, uf, disp);
 
 /*                             PROFILE_CYCLE_START; */
                        }
@@ -1527,6 +1544,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
                                }
                                break;
                        }
+                       codegen_emit_patchable_barrier(iptr, cd, pr, fi);
                        break;
 
                case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
@@ -1585,7 +1603,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 
 /*                             PROFILE_CYCLE_STOP; */
 
-                               patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
+                               pr = patcher_add_patch_ref(jd, PATCHER_get_putfield, uf, 0);
 
 /*                             PROFILE_CYCLE_START; */
                        } 
@@ -1611,6 +1629,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
                                M_DST32(s2, s1, disp);
                                break;
                        }
+                       codegen_emit_patchable_barrier(iptr, cd, pr, fi);
                        break;
 
                case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
@@ -1626,7 +1645,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
 
 /*                             PROFILE_CYCLE_STOP; */
 
-                               patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
+                               pr = patcher_add_patch_ref(jd, PATCHER_putfieldconst, uf, 0);
 
 /*                             PROFILE_CYCLE_START; */
                        } 
@@ -1653,6 +1672,7 @@ void codegen_emit_instruction(jitdata* jd, instruction* iptr)
                                        M_LST32(REG_ITMP2, s1, disp);
                                break;
                        }
+                       codegen_emit_patchable_barrier(iptr, cd, pr, fi);
                        break;
 
 

diff --git a/src/vm/jit/x86_64/codegen.h b/src/vm/jit/x86_64/codegen.h
index 4fa215d54d72f7cd4fed04f964900501a2edda81..086502be5a2748742f75bf321558ae679d8765ec 100644 (file)
--- a/src/vm/jit/x86_64/codegen.h
+++ b/src/vm/jit/x86_64/codegen.h
@@ -82,6 +82,9 @@
 #define BRANCH_UNCONDITIONAL_SIZE    5  /* size in bytes of a branch          */
 #define BRANCH_CONDITIONAL_SIZE      6  /* size in bytes of a branch          */
 
+/* These NOPs are never executed; they are only used as placeholders during
+ * code generation.
+ */
 #define BRANCH_NOPS \
     do { \
         M_NOP; \

diff --git a/src/vm/jit/x86_64/patcher.c b/src/vm/jit/x86_64/patcher.c
index 81cc09e0e8bbc78583471327af0d14c9203634c0..be4fd4e7a8c447f936875e22e0a07d59a280a5bf 100644 (file)
--- a/src/vm/jit/x86_64/patcher.c
+++ b/src/vm/jit/x86_64/patcher.c
@@ -74,6 +74,25 @@ bool patcher_is_valid_trap_instruction_at(void* pc)
        return (mcode == 0x0b0f);
 }
 
+/**
+ * Overwrites the MFENCE instruction at the indicated address with a 3-byte
+ * NOP. The MFENCE instruction is not allowed to cross a (4-byte) word
+ * boundary.
+ *
+ * @param pc Program counter.
+ */
+static void patch_out_mfence(void *pc)
+{
+       uint32_t *p = (uint32_t*) (((uintptr_t) pc) & ~3);
+
+       assert((((uintptr_t) pc) & 3) < 2);
+       if (((uintptr_t) pc) & 1)
+               *p = *p & 0x000000ff | 0x001f0f00;
+       else
+               *p = *p & 0xff000000 | 0x00001f0f;
+
+       md_icacheflush(p, 4);
+}
 
 /* patcher_resolve_classref_to_classinfo ***************************************
 
@@ -225,6 +244,9 @@ bool patcher_get_putstatic(patchref_t *pr)
        // Patch the field value's address.
        *datap = (uintptr_t) fi->value;
 
+       if (pr->disp_mb && !(fi->flags & ACC_VOLATILE))
+               patch_out_mfence(ra + pr->disp_mb - 2);
+
        // Synchronize data cache.
        md_dcacheflush((void*) pr->datap, SIZEOF_VOID_P);
 
@@ -274,6 +296,9 @@ bool patcher_get_putfield(patchref_t *pr)
                        *((int32_t*) (pc + 5)) = fi->offset;
        }
 
+       if (pr->disp_mb && !(fi->flags & ACC_VOLATILE))
+               patch_out_mfence(pc + pr->disp_mb - 2);
+
        // Synchronize instruction cache.
        md_icacheflush(pc, 6 + sizeof(int32_t));
 
@@ -322,6 +347,9 @@ bool patcher_putfieldconst(patchref_t *pr)
                        *((uint32_t*) (pc + 3)) = fi->offset;
        }
 
+       if (pr->disp_mb && !(fi->flags & ACC_VOLATILE))
+               patch_out_mfence(pc + pr->disp_mb - 2);
+
        // Synchronize instruction cache.
        md_icacheflush(pc, 14 + sizeof(int32_t));