* Removed all Id tags.
[cacao.git] / src / vm / jit / i386 / emit.c
index 92ad9fe2ea4807714ee39d86eae36a3c737c0b71..9e1f42d14c8023b5082404f089dd73a085f84c95 100644 (file)
@@ -1,6 +1,6 @@
 /* src/vm/jit/i386/emit.c - i386 code emitter functions
 
-   Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
+   Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
    J. Wenninger, Institut f. Computersprachen - TU Wien
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.
 
-   Contact: cacao@cacaojvm.org
-
-   Authors: Christian Thalinger
-
-   Changes:
-
-   $Id: emit.c 5277 2006-08-25 07:29:05Z twisti $
-
 */
 
 
 
 #include "vm/types.h"
 
-#include "vm/jit/i386/md-abi.h"
-#include "vm/jit/i386/md-emit.h"
 #include "vm/jit/i386/codegen.h"
+#include "vm/jit/i386/emit.h"
+#include "vm/jit/i386/md-abi.h"
 
-#if defined(ENABLE_THREADS)
-# include "threads/native/lock.h"
-#endif
+#include "mm/memory.h"
+
+#include "threads/lock-common.h"
 
-#include "vm/statistics.h"
+#include "vm/builtin.h"
+#include "vm/exceptions.h"
+
+#include "vm/jit/abi.h"
 #include "vm/jit/asmpart.h"
 #include "vm/jit/dseg.h"
-#include "vm/jit/emit.h"
+#include "vm/jit/emit-common.h"
 #include "vm/jit/jit.h"
+#include "vm/jit/replace.h"
+
+#include "vmcore/options.h"
+#include "vmcore/statistics.h"
 
 
-/* emit_load_s1 ****************************************************************
+/* emit_load ******************************************************************
 
-   Emits a possible load of the first source operand.
+   Emits a possible load of an operand.
 
 *******************************************************************************/
 
-s4 emit_load_s1(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+inline s4 emit_load(jitdata *jd, instruction *iptr, varinfo *src, s4 tempreg)
 {
        codegendata  *cd;
        s4            disp;
@@ -70,83 +69,80 @@ s4 emit_load_s1(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
 
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
+       if (IS_INMEMORY(src->flags)) {
                COUNT_SPILLS;
 
-               disp = src->regoff * 4;
-
-               if (IS_FLT_DBL_TYPE(src->type)) {
-                       if (IS_2_WORD_TYPE(src->type))
-                               M_DLD(tempreg, REG_SP, disp);
-                       else
-                               M_FLD(tempreg, REG_SP, disp);
-               }
-               else {
-                       if (IS_2_WORD_TYPE(src->type))
-                               M_LLD(tempreg, REG_SP, disp);
-                       else
-                               M_ILD(tempreg, REG_SP, disp);
+               disp = src->vv.regoff;
+
+               switch (src->type) {
+               case TYPE_INT:
+               case TYPE_ADR:
+                       M_ILD(tempreg, REG_SP, disp);
+                       break;
+               case TYPE_LNG:
+                       M_LLD(tempreg, REG_SP, disp);
+                       break;
+               case TYPE_FLT:
+                       M_FLD(tempreg, REG_SP, disp);
+                       break;
+               case TYPE_DBL:
+                       M_DLD(tempreg, REG_SP, disp);
+                       break;
+               default:
+                       vm_abort("emit_load: unknown type %d", src->type);
                }
 
                reg = tempreg;
        }
        else
-               reg = src->regoff;
+               reg = src->vv.regoff;
 
        return reg;
 }
 
 
-/* emit_load_s2 ****************************************************************
+/* emit_load_low ************************************************************
 
-   Emits a possible load of the second source operand.
+   Emits a possible load of the low 32-bits of an operand.
 
 *******************************************************************************/
 
-s4 emit_load_s2(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+inline s4 emit_load_low(jitdata *jd, instruction *iptr, varinfo *src,s4 tempreg)
 {
        codegendata  *cd;
        s4            disp;
        s4            reg;
 
+       assert(src->type == TYPE_LNG);
+
        /* get required compiler data */
 
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
+
+       if (IS_INMEMORY(src->flags)) {
                COUNT_SPILLS;
 
-               disp = src->regoff * 4;
+               disp = src->vv.regoff;
 
-               if (IS_FLT_DBL_TYPE(src->type)) {
-                       if (IS_2_WORD_TYPE(src->type))
-                               M_DLD(tempreg, REG_SP, disp);
-                       else
-                               M_FLD(tempreg, REG_SP, disp);
-               }
-               else {
-                       if (IS_2_WORD_TYPE(src->type))
-                               M_LLD(tempreg, REG_SP, disp);
-                       else
-                               M_ILD(tempreg, REG_SP, disp);
-               }
+               M_ILD(tempreg, REG_SP, disp);
 
                reg = tempreg;
        }
        else
-               reg = src->regoff;
+               reg = GET_LOW_REG(src->vv.regoff);
 
        return reg;
 }
 
 
-/* emit_load_s3 ****************************************************************
+/* emit_load_high ***********************************************************
 
-   Emits a possible load of the third source operand.
+   Emits a possible load of the high 32-bits of an operand.
 
 *******************************************************************************/
 
-s4 emit_load_s3(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+inline s4 emit_load_high(jitdata *jd, instruction *iptr,varinfo *src,s4 tempreg)
 {
        codegendata  *cd;
        s4            disp;
@@ -154,373 +150,342 @@ s4 emit_load_s3(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
 
        /* get required compiler data */
 
+       assert(src->type == TYPE_LNG);
+
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
+       if (IS_INMEMORY(src->flags)) {
                COUNT_SPILLS;
 
-               disp = src->regoff * 4;
+               disp = src->vv.regoff;
 
-               if (IS_FLT_DBL_TYPE(src->type)) {
-                       if (IS_2_WORD_TYPE(src->type))
-                               M_DLD(tempreg, REG_SP, disp);
-                       else
-                               M_FLD(tempreg, REG_SP, disp);
-               }
-               else {
-                       if (IS_2_WORD_TYPE(src->type))
-                               M_LLD(tempreg, REG_SP, disp);
-                       else
-                               M_ILD(tempreg, REG_SP, disp);
-               }
+               M_ILD(tempreg, REG_SP, disp + 4);
 
                reg = tempreg;
        }
        else
-               reg = src->regoff;
+               reg = GET_HIGH_REG(src->vv.regoff);
 
        return reg;
 }
 
 
-/* emit_load_s1_low ************************************************************
+/* emit_store ******************************************************************
 
-   Emits a possible load of the low 32-bits of the first long source
-   operand.
+   Emits a possible store of the destination operand.
 
 *******************************************************************************/
 
-s4 emit_load_s1_low(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+inline void emit_store(jitdata *jd, instruction *iptr, varinfo *dst, s4 d)
 {
        codegendata  *cd;
        s4            disp;
-       s4            reg;
-
-       assert(src->type == TYPE_LNG);
 
        /* get required compiler data */
 
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
+       if (IS_INMEMORY(dst->flags)) {
                COUNT_SPILLS;
 
-               disp = src->regoff * 4;
-
-               M_ILD(tempreg, REG_SP, disp);
-
-               reg = tempreg;
+               disp = dst->vv.regoff;
+
+               switch (dst->type) {
+               case TYPE_INT:
+               case TYPE_ADR:
+                       M_IST(d, REG_SP, disp);
+                       break;
+               case TYPE_LNG:
+                       M_LST(d, REG_SP, disp);
+                       break;
+               case TYPE_FLT:
+                       M_FST(d, REG_SP, disp);
+                       break;
+               case TYPE_DBL:
+                       M_DST(d, REG_SP, disp);
+                       break;
+               default:
+                       vm_abort("emit_store: unknown type %d", dst->type);
+               }
        }
-       else
-               reg = GET_LOW_REG(src->regoff);
-
-       return reg;
 }
 
 
-/* emit_load_s2_low ************************************************************
+/* emit_store_low **************************************************************
 
-   Emits a possible load of the low 32-bits of the second long source
+   Emits a possible store of the low 32-bits of the destination
    operand.
 
 *******************************************************************************/
 
-s4 emit_load_s2_low(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+inline void emit_store_low(jitdata *jd, instruction *iptr, varinfo *dst, s4 d)
 {
        codegendata  *cd;
-       s4            disp;
-       s4            reg;
 
-       assert(src->type == TYPE_LNG);
+       assert(dst->type == TYPE_LNG);
 
        /* get required compiler data */
 
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
+       if (IS_INMEMORY(dst->flags)) {
                COUNT_SPILLS;
-
-               disp = src->regoff * 4;
-
-               M_ILD(tempreg, REG_SP, disp);
-
-               reg = tempreg;
+               M_IST(GET_LOW_REG(d), REG_SP, dst->vv.regoff);
        }
-       else
-               reg = GET_LOW_REG(src->regoff);
-
-       return reg;
 }
 
 
-/* emit_load_s1_high ***********************************************************
+/* emit_store_high *************************************************************
 
-   Emits a possible load of the high 32-bits of the first long source
+   Emits a possible store of the high 32-bits of the destination
    operand.
 
 *******************************************************************************/
 
-s4 emit_load_s1_high(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+inline void emit_store_high(jitdata *jd, instruction *iptr, varinfo *dst, s4 d)
 {
        codegendata  *cd;
-       s4            disp;
-       s4            reg;
 
-       assert(src->type == TYPE_LNG);
+       assert(dst->type == TYPE_LNG);
 
        /* get required compiler data */
 
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
+       if (IS_INMEMORY(dst->flags)) {
                COUNT_SPILLS;
-
-               disp = src->regoff * 4;
-
-               M_ILD(tempreg, REG_SP, disp + 4);
-
-               reg = tempreg;
+               M_IST(GET_HIGH_REG(d), REG_SP, dst->vv.regoff + 4);
        }
-       else
-               reg = GET_HIGH_REG(src->regoff);
-
-       return reg;
 }
 
 
-/* emit_load_s2_high ***********************************************************
+/* emit_copy *******************************************************************
 
-   Emits a possible load of the high 32-bits of the second long source
-   operand.
+   Generates a register/memory to register/memory copy.
 
 *******************************************************************************/
 
-s4 emit_load_s2_high(jitdata *jd, instruction *iptr, stackptr src, s4 tempreg)
+void emit_copy(jitdata *jd, instruction *iptr)
 {
-       codegendata  *cd;
-       s4            disp;
-       s4            reg;
-
-       assert(src->type == TYPE_LNG);
+       codegendata *cd;
+       varinfo     *src;
+       varinfo     *dst;
+       s4           s1, d;
 
        /* get required compiler data */
 
        cd = jd->cd;
 
-       if (src->flags & INMEMORY) {
-               COUNT_SPILLS;
-
-               disp = src->regoff * 4;
-
-               M_ILD(tempreg, REG_SP, disp + 4);
-
-               reg = tempreg;
-       } else
-               reg = GET_HIGH_REG(src->regoff);
-
-       return reg;
-}
-
-
-/* emit_store ******************************************************************
-
-   Emits a possible store of the destination operand.
+       /* get source and destination variables */
 
-*******************************************************************************/
+       src = VAROP(iptr->s1);
+       dst = VAROP(iptr->dst);
 
-void emit_store(jitdata *jd, instruction *iptr, stackptr dst, s4 d)
-{
-       codegendata  *cd;
-
-       /* get required compiler data */
+       if ((src->vv.regoff != dst->vv.regoff) ||
+               ((src->flags ^ dst->flags) & INMEMORY)) {
 
-       cd = jd->cd;
+               if ((src->type == TYPE_RET) || (dst->type == TYPE_RET)) {
+                       /* emit nothing, as the value won't be used anyway */
+                       return;
+               }
 
-       if (dst->flags & INMEMORY) {
-               COUNT_SPILLS;
+               /* If one of the variables resides in memory, we can eliminate
+                  the register move from/to the temporary register with the
+                  order of getting the destination register and the load. */
 
-               if (IS_FLT_DBL_TYPE(dst->type)) {
-                       if (IS_2_WORD_TYPE(dst->type))
-                               M_DST(d, REG_SP, dst->regoff * 4);
+               if (IS_INMEMORY(src->flags)) {
+                       if (IS_LNG_TYPE(src->type))
+                               d = codegen_reg_of_var(iptr->opc, dst, REG_ITMP12_PACKED);
                        else
-                               M_FST(d, REG_SP, dst->regoff * 4);
+                               d = codegen_reg_of_var(iptr->opc, dst, REG_ITMP1);
+
+                       s1 = emit_load(jd, iptr, src, d);
                }
                else {
-                       if (IS_2_WORD_TYPE(dst->type))
-                               M_LST(d, REG_SP, dst->regoff * 4);
+                       if (IS_LNG_TYPE(src->type))
+                               s1 = emit_load(jd, iptr, src, REG_ITMP12_PACKED);
                        else
-                               M_IST(d, REG_SP, dst->regoff * 4);
+                               s1 = emit_load(jd, iptr, src, REG_ITMP1);
+
+                       d = codegen_reg_of_var(iptr->opc, dst, s1);
+               }
+
+               if (s1 != d) {
+                       switch (src->type) {
+                       case TYPE_INT:
+                       case TYPE_ADR:
+                               M_MOV(s1, d);
+                               break;
+                       case TYPE_LNG:
+                               M_LNGMOVE(s1, d);
+                               break;
+                       case TYPE_FLT:
+                       case TYPE_DBL:
+/*                             M_FMOV(s1, d); */
+                               break;
+                       default:
+                               vm_abort("emit_copy: unknown type %d", src->type);
+                       }
                }
+
+               emit_store(jd, iptr, dst, d);
        }
 }
 
 
-/* emit_store_low **************************************************************
+/* emit_branch *****************************************************************
 
-   Emits a possible store of the low 32-bits of the destination
-   operand.
+   Emits the code for conditional and unconditional branchs.
 
 *******************************************************************************/
 
-void emit_store_low(jitdata *jd, instruction *iptr, stackptr dst, s4 d)
+void emit_branch(codegendata *cd, s4 disp, s4 condition, s4 reg, u4 options)
 {
-       codegendata  *cd;
+       s4 branchdisp;
 
-       assert(dst->type == TYPE_LNG);
+       /* ATTENTION: a displacement overflow cannot happen */
 
-       /* get required compiler data */
+       /* check which branch to generate */
 
-       cd = jd->cd;
+       if (condition == BRANCH_UNCONDITIONAL) {
 
-       if (dst->flags & INMEMORY) {
-               COUNT_SPILLS;
-               M_IST(GET_LOW_REG(d), REG_SP, dst->regoff * 4);
+               /* calculate the different displacements */
+
+               branchdisp = disp - BRANCH_UNCONDITIONAL_SIZE;
+
+               M_JMP_IMM(branchdisp);
+       }
+       else {
+               /* calculate the different displacements */
+
+               branchdisp = disp - BRANCH_CONDITIONAL_SIZE;
+
+               switch (condition) {
+               case BRANCH_EQ:
+                       M_BEQ(branchdisp);
+                       break;
+               case BRANCH_NE:
+                       M_BNE(branchdisp);
+                       break;
+               case BRANCH_LT:
+                       M_BLT(branchdisp);
+                       break;
+               case BRANCH_GE:
+                       M_BGE(branchdisp);
+                       break;
+               case BRANCH_GT:
+                       M_BGT(branchdisp);
+                       break;
+               case BRANCH_LE:
+                       M_BLE(branchdisp);
+                       break;
+               case BRANCH_ULT:
+                       M_BB(branchdisp);
+                       break;
+               case BRANCH_ULE:
+                       M_BBE(branchdisp);
+                       break;
+               case BRANCH_UGE:
+                       M_BAE(branchdisp);
+                       break;
+               case BRANCH_UGT:
+                       M_BA(branchdisp);
+                       break;
+               default:
+                       vm_abort("emit_branch: unknown condition %d", condition);
+               }
        }
 }
 
 
-/* emit_store_high *************************************************************
+/* emit_arithmetic_check *******************************************************
 
-   Emits a possible store of the high 32-bits of the destination
-   operand.
+   Emit an ArithmeticException check.
 
 *******************************************************************************/
 
-void emit_store_high(jitdata *jd, instruction *iptr, stackptr dst, s4 d)
+void emit_arithmetic_check(codegendata *cd, instruction *iptr, s4 reg)
 {
-       codegendata  *cd;
-
-       assert(dst->type == TYPE_LNG);
-
-       /* get required compiler data */
-
-       cd = jd->cd;
-
-       if (dst->flags & INMEMORY) {
-               COUNT_SPILLS;
-               M_IST(GET_HIGH_REG(d), REG_SP, dst->regoff * 4 + 4);
+       if (INSTRUCTION_MUST_CHECK(iptr)) {
+               M_TEST(reg);
+               M_BNE(6);
+               M_ALD_MEM(reg, EXCEPTION_HARDWARE_ARITHMETIC);
        }
 }
 
 
-/* emit_copy *******************************************************************
+/* emit_arrayindexoutofbounds_check ********************************************
 
-   XXX
+   Emit a ArrayIndexOutOfBoundsException check.
 
 *******************************************************************************/
 
-void emit_copy(jitdata *jd, instruction *iptr, stackptr src, stackptr dst)
+void emit_arrayindexoutofbounds_check(codegendata *cd, instruction *iptr, s4 s1, s4 s2)
 {
-       codegendata  *cd;
-       registerdata *rd;
-       s4            s1, d;
+       if (INSTRUCTION_MUST_CHECK(iptr)) {
+        M_ILD(REG_ITMP3, s1, OFFSET(java_array_t, size));
+        M_CMP(REG_ITMP3, s2);
+        M_BB(6);
+               M_ALD_MEM(s2, EXCEPTION_HARDWARE_ARRAYINDEXOUTOFBOUNDS);
+       }
+}
 
-       /* get required compiler data */
 
-       cd = jd->cd;
-       rd = jd->rd;
+/* emit_classcast_check ********************************************************
 
-       if ((src->regoff != dst->regoff) ||
-               ((src->flags ^ dst->flags) & INMEMORY)) {
-               if (IS_LNG_TYPE(src->type))
-                       d = codegen_reg_of_var(rd, iptr->opc, dst, REG_ITMP12_PACKED);
-               else
-                       d = codegen_reg_of_var(rd, iptr->opc, dst, REG_ITMP1);
+   Emit a ClassCastException check.
 
-               s1 = emit_load_s1(jd, iptr, src, d);
+*******************************************************************************/
 
-               if (s1 != d) {
-                       if (IS_FLT_DBL_TYPE(src->type)) {
-/*                             M_FMOV(s1, d); */
-                       } else {
-                               if (IS_2_WORD_TYPE(src->type))
-                                       M_LNGMOVE(s1, d);
-                               else
-                    M_MOV(s1, d);
-                       }
+void emit_classcast_check(codegendata *cd, instruction *iptr, s4 condition, s4 reg, s4 s1)
+{
+       if (INSTRUCTION_MUST_CHECK(iptr)) {
+               switch (condition) {
+               case BRANCH_LE:
+                       M_BGT(6);
+                       break;
+               case BRANCH_EQ:
+                       M_BNE(6);
+                       break;
+               case BRANCH_ULE:
+                       M_BBE(6);
+                       break;
+               default:
+                       vm_abort("emit_classcast_check: unknown condition %d", condition);
                }
-
-               emit_store(jd, iptr, dst, d);
+               M_ALD_MEM(s1, EXCEPTION_HARDWARE_CLASSCAST);
        }
 }
 
 
-/* emit_exception_stubs ********************************************************
+/* emit_nullpointer_check ******************************************************
 
-   Generates the code for the exception stubs.
+   Emit a NullPointerException check.
 
 *******************************************************************************/
 
-void emit_exception_stubs(jitdata *jd)
+void emit_nullpointer_check(codegendata *cd, instruction *iptr, s4 reg)
 {
-       codegendata  *cd;
-       registerdata *rd;
-       exceptionref *eref;
-       s4            targetdisp;
-
-       /* get required compiler data */
-
-       cd = jd->cd;
-       rd = jd->rd;
-
-       /* generate exception stubs */
-
-       targetdisp = 0;
-
-       for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
-               gen_resolvebranch(cd->mcodebase + eref->branchpos,
-                                                 eref->branchpos,
-                                                 cd->mcodeptr - cd->mcodebase);
-
-               MCODECHECK(512);
-
-               /* Check if the exception is an
-                  ArrayIndexOutOfBoundsException.  If so, move index register
-                  into REG_ITMP1. */
-
-               if (eref->reg != -1)
-                       M_INTMOVE(eref->reg, REG_ITMP1);
-
-               /* calcuate exception address */
-
-               M_MOV_IMM(0, REG_ITMP2_XPC);
-               dseg_adddata(cd);
-               M_AADD_IMM32(eref->branchpos - 6, REG_ITMP2_XPC);
-
-               /* move function to call into REG_ITMP3 */
-
-               M_MOV_IMM(eref->function, REG_ITMP3);
-
-               if (targetdisp == 0) {
-                       targetdisp = cd->mcodeptr - cd->mcodebase;
-
-                       M_ASUB_IMM(5 * 4, REG_SP);
-
-                       /* first store REG_ITMP1 so we can use it */
+       if (INSTRUCTION_MUST_CHECK(iptr)) {
+               M_TEST(reg);
+               M_BNE(6);
+               M_ALD_MEM(reg, EXCEPTION_HARDWARE_NULLPOINTER);
+       }
+}
 
-                       M_AST(REG_ITMP1, REG_SP, 4 * 4);                    /* for AIOOBE */
 
-                       M_AST_IMM(0, REG_SP, 0 * 4);
-                       dseg_adddata(cd);
-                       M_MOV(REG_SP, REG_ITMP1);
-                       M_AADD_IMM(5 * 4, REG_ITMP1);
-                       M_AST(REG_ITMP1, REG_SP, 1 * 4);
-                       M_ALD(REG_ITMP1, REG_SP, (5 + cd->stackframesize) * 4);
-                       M_AST(REG_ITMP1, REG_SP, 2 * 4);
-                       M_AST(REG_ITMP2_XPC, REG_SP, 3 * 4);
+/* emit_exception_check ********************************************************
 
-                       M_CALL(REG_ITMP3);
+   Emit an Exception check.
 
-                       M_ALD(REG_ITMP2_XPC, REG_SP, 3 * 4);
-                       M_AADD_IMM(5 * 4, REG_SP);
+*******************************************************************************/
 
-                       M_MOV_IMM(asm_handle_exception, REG_ITMP3);
-                       M_JMP(REG_ITMP3);
-               }
-               else {
-                       M_JMP_IMM((cd->mcodebase + targetdisp) -
-                                         (cd->mcodeptr + PATCHER_CALL_SIZE));
-               }
+void emit_exception_check(codegendata *cd, instruction *iptr)
+{
+       if (INSTRUCTION_MUST_CHECK(iptr)) {
+               M_TEST(REG_RESULT);
+               M_BNE(6);
+               M_ALD_MEM(REG_RESULT, EXCEPTION_HARDWARE_EXCEPTION);
        }
 }
 
@@ -576,9 +541,9 @@ void emit_patcher_stubs(jitdata *jd)
                /* move pointer to java_objectheader onto stack */
 
 #if defined(ENABLE_THREADS)
-               (void) dseg_addaddress(cd, NULL);                          /* flcword */
-               (void) dseg_addaddress(cd, lock_get_initial_lock_word());
-               disp = dseg_addaddress(cd, NULL);                          /* vftbl   */
+               (void) dseg_add_unique_address(cd, NULL);                  /* flcword */
+               (void) dseg_add_unique_address(cd, lock_get_initial_lock_word());
+               disp = dseg_add_unique_address(cd, NULL);                  /* vftbl   */
 
                M_MOV_IMM(0, REG_ITMP3);
                dseg_adddata(cd);
@@ -609,53 +574,173 @@ void emit_patcher_stubs(jitdata *jd)
 }
 
 
-/* emit_replacement_stubs ******************************************************
+/* emit_trap *******************************************************************
 
-   Generates the code for the replacement stubs.
+   Emit a trap instruction and return the original machine code.
 
 *******************************************************************************/
 
-void emit_replacement_stubs(jitdata *jd)
+uint32_t emit_trap(codegendata *cd)
 {
-       codegendata *cd;
-       codeinfo    *code;
-       rplpoint    *rplp;
-       s4           disp;
-       s4           i;
+       uint32_t mcode;
+
+       /* Get machine code which is patched back in later. The
+          trap is 1 instruction word long. */
+
+       mcode = *((uint32_t *) cd->mcodeptr);
+
+       M_NOP;
+
+       return mcode;
+}
+
+
+/* emit_verbosecall_enter ******************************************************
+
+   Generates the code for the call trace.
+
+*******************************************************************************/
+
+#if !defined(NDEBUG)
+void emit_verbosecall_enter(jitdata *jd)
+{
+       methodinfo   *m;
+       codegendata  *cd;
+       registerdata *rd;
+       methoddesc   *md;
+       int32_t       disp;
+       int           i;
+       int           d;
+
+       if (!JITDATA_HAS_FLAG_VERBOSECALL(jd))
+               return;
 
        /* get required compiler data */
 
-       cd   = jd->cd;
-       code = jd->code;
+       m  = jd->m;
+       cd = jd->cd;
+       rd = jd->rd;
 
-       rplp = code->rplpoints;
+       md = m->parseddesc;
 
-       for (i = 0; i < code->rplpointcount; ++i, ++rplp) {
-               /* check code segment size */
+       /* mark trace code */
 
-               MCODECHECK(512);
+       M_NOP;
 
-               /* note start of stub code */
+       /* methodinfo* + arguments + return address */
 
-               rplp->outcode = (u1 *) (ptrint) (cd->mcodeptr - cd->mcodebase);
+       disp = (TRACE_ARGS_NUM + 1 + TMP_CNT) * 8 + cd->stackframesize * 8 + 4;
 
-               /* make machine code for patching */
+       M_ASUB_IMM((TRACE_ARGS_NUM + 1 + TMP_CNT) * 8, REG_SP);
 
-               disp = (ptrint) (rplp->outcode - rplp->pc) - 5;
+       /* save temporary registers for leaf methods */
 
-               rplp->mcode = 0xe9 | ((u8) disp << 8);
+       for (i = 0; i < INT_TMP_CNT; i++)
+               M_IST(rd->tmpintregs[i], REG_SP, (TRACE_ARGS_NUM + 1 + i) * 8);
 
-               /* push address of `rplpoint` struct */
-                       
-               M_PUSH_IMM(rplp);
+       /* save argument registers */
 
-               /* jump to replacement function */
+       for (i = 0; i < md->paramcount; i++) {
+               d = i * 8;
 
-               M_PUSH_IMM(asm_replacement_out);
-               M_RET;
+               switch (md->paramtypes[i].type) {
+               case TYPE_INT:
+                       M_ILD(EAX, REG_SP, disp);
+                       emit_cltd(cd);
+                       M_LST(EAX_EDX_PACKED, REG_SP, d);
+                       break;
+               case TYPE_LNG:
+                       M_LLD(REG_ITMP12_PACKED, REG_SP, disp);
+                       M_LST(REG_ITMP12_PACKED, REG_SP, d);
+                       break;
+               case TYPE_ADR:
+                       M_ALD(REG_ITMP1, REG_SP, disp);
+                       M_AST(REG_ITMP1, REG_SP, d);
+                       M_IST_IMM(0, REG_SP, d + 4);                /* high-bits are zero */
+                       break;
+               case TYPE_FLT:
+                       M_FLD(REG_NULL, REG_SP, disp);
+                       M_FST(REG_NULL, REG_SP, d);
+                       M_IST_IMM(0, REG_SP, d + 4);                /* high-bits are zero */
+                       break;
+               case TYPE_DBL:
+                       M_DLD(REG_NULL, REG_SP, disp);
+                       M_DST(REG_NULL, REG_SP, d);
+                       break;
+               }
+
+               disp += 8;
        }
+
+       M_AST_IMM(m, REG_SP, TRACE_ARGS_NUM * 8);
+
+       M_MOV_IMM(builtin_verbosecall_enter, REG_ITMP1);
+       M_CALL(REG_ITMP1);
+
+       /* restore temporary registers for leaf methods */
+
+       for (i = 0; i < INT_TMP_CNT; i++)
+               M_ILD(rd->tmpintregs[i], REG_SP, (TRACE_ARGS_NUM + 1 + i) * 8);
+
+       M_AADD_IMM((TRACE_ARGS_NUM + 1 + TMP_CNT) * 8, REG_SP);
+
+       /* mark trace code */
+
+       M_NOP;
 }
-       
+#endif /* !defined(NDEBUG) */
+
+
+/* emit_verbosecall_exit *******************************************************
+
+   Generates the code for the call trace.
+
+   void builtin_verbosecall_exit(s8 l, double d, float f, methodinfo *m);
+
+*******************************************************************************/
+
+#if !defined(NDEBUG)
+void emit_verbosecall_exit(jitdata *jd)
+{
+       methodinfo   *m;
+       codegendata  *cd;
+       registerdata *rd;
+
+       if (!JITDATA_HAS_FLAG_VERBOSECALL(jd))
+               return;
+
+       /* get required compiler data */
+
+       m  = jd->m;
+       cd = jd->cd;
+       rd = jd->rd;
+
+       /* mark trace code */
+
+       M_NOP;
+
+       M_ASUB_IMM(8 + 8 + 4 + 4 + 8, REG_SP);  /* +8: keep stack 16-byte aligned */
+
+       M_LST(REG_RESULT_PACKED, REG_SP, 0 * 8);
+
+       M_DSTNP(REG_NULL, REG_SP, 1 * 8);
+       M_FSTNP(REG_NULL, REG_SP, 2 * 8);
+
+       M_AST_IMM(m, REG_SP, 2 * 8 + 1 * 4);
+
+       M_MOV_IMM(builtin_verbosecall_exit, REG_ITMP1);
+       M_CALL(REG_ITMP1);
+
+       M_LLD(REG_RESULT_PACKED, REG_SP, 0 * 4);
+
+       M_AADD_IMM(8 + 8 + 4 + 4 + 8, REG_SP);
+
+       /* mark trace code */
+
+       M_NOP;
+}
+#endif /* !defined(NDEBUG) */
+
 
 /* code generation functions **************************************************/
 
@@ -999,6 +1084,20 @@ void emit_alu_imm_membase(codegendata *cd, s4 opc, s4 imm, s4 basereg, s4 disp)
 }
 
 
+void emit_alu_imm_memabs(codegendata *cd, s4 opc, s4 imm, s4 disp)
+{
+       if (IS_IMM8(imm)) { 
+               *(cd->mcodeptr++) = 0x83;
+               emit_mem(opc, disp);
+               emit_imm8((imm));
+       } else { 
+               *(cd->mcodeptr++) = 0x81;
+               emit_mem(opc, disp);
+               emit_imm32((imm));
+       }
+}
+
+
 void emit_test_reg_reg(codegendata *cd, s4 reg, s4 dreg)
 {
        *(cd->mcodeptr++) = 0x85;