X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=src%2Fvm%2Fjit%2Fx86_64%2Femit.c;h=7a4c65a39d20537e2ec1d99604208695a9634156;hb=d1377204f560df4d6656f58914bc4d242193c5e0;hp=fd7df3364f14a8a2e45789b1640443b6ef764033;hpb=0c006b0b5ce06dfa211b305bdc0bf1672b77801b;p=cacao.git

diff --git a/src/vm/jit/x86_64/emit.c b/src/vm/jit/x86_64/emit.c
index fd7df3364..7a4c65a39 100644
--- a/src/vm/jit/x86_64/emit.c
+++ b/src/vm/jit/x86_64/emit.c
@@ -1,9 +1,7 @@
 /* src/vm/jit/x86_64/emit.c - x86_64 code emitter functions
 
-   Copyright (C) 1996-2005, 2006, 2007 R. Grafl, A. Krall, C. Kruegel,
-   C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
-   E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
-   J. Wenninger, Institut f. Computersprachen - TU Wien
+   Copyright (C) 1996-2005, 2006, 2007, 2008, 2009
+   CACAOVM - Verein zur Foerderung der freien virtuellen Maschine CACAO
 
    This file is part of CACAO.
 
@@ -29,28 +27,29 @@
 #include <assert.h>
 
 #include "vm/types.h"
+#include "vm/os.hpp"
 
 #include "md-abi.h"
 
 #include "vm/jit/x86_64/codegen.h"
 #include "vm/jit/x86_64/emit.h"
 
-#include "mm/memory.h"
+#include "mm/memory.hpp"
 
-#include "threads/lock-common.h"
+#include "threads/lock.hpp"
 
-#include "vm/builtin.h"
-#include "vm/exceptions.h"
+#include "vm/options.h"
 
 #include "vm/jit/abi.h"
 #include "vm/jit/abi-asm.h"
 #include "vm/jit/asmpart.h"
-#include "vm/jit/codegen-common.h"
-#include "vm/jit/emit-common.h"
-#include "vm/jit/jit.h"
-#include "vm/jit/replace.h"
-
-#include "vmcore/options.h"
+#include "vm/jit/codegen-common.hpp"
+#include "vm/jit/emit-common.hpp"
+#include "vm/jit/jit.hpp"
+#include "vm/jit/patcher-common.hpp"
+#include "vm/jit/replace.hpp"
+#include "vm/jit/trace.hpp"
+#include "vm/jit/trap.hpp"
 
 
 /* emit_load *******************************************************************
@@ -110,42 +109,15 @@ s4 emit_load(jitdata *jd, instruction *iptr, varinfo *src, s4 tempreg)
     
 *******************************************************************************/
 
-inline void emit_store(jitdata *jd, instruction *iptr, varinfo *dst, s4 d)
+void emit_store(jitdata *jd, instruction *iptr, varinfo *dst, s4 d)
 {
 	codegendata  *cd;
 	s4            disp;
-#if 0
-	s4            s;
-	u2            opcode;
-#endif
 
 	/* get required compiler data */
 
 	cd = jd->cd;
 
-#if 0
-	/* do we have to generate a conditional move? */
-
-	if ((iptr != NULL) && (iptr->opc & ICMD_CONDITION_MASK)) {
-		/* the passed register d is actually the source register */
-
-		s = d;
-
-		/* Only pass the opcode to codegen_reg_of_var to get the real
-		   destination register. */
-
-		opcode = iptr->opc & ICMD_OPCODE_MASK;
-
-		/* get the real destination register */
-
-		d = codegen_reg_of_var(rd, opcode, dst, REG_ITMP1);
-
-		/* and emit the conditional move */
-
-		emit_cmovxx(cd, iptr, s, d);
-	}
-#endif
-
 	if (IS_INMEMORY(dst->flags)) {
 		COUNT_SPILLS;
 
@@ -261,6 +233,16 @@ void emit_cmovxx(codegendata *cd, instruction *iptr, s4 s, s4 d)
 }
 
 
+/**
+ * Emits code updating the condition register by comparing one integer
+ * register to an immediate integer value.
+ */
+void emit_icmp_imm(codegendata* cd, int reg, int32_t value)
+{
+	M_ICMP_IMM(value, reg);
+}
+
+
 /* emit_branch *****************************************************************
 
    Emits the code for conditional and unconditional branchs.
@@ -337,7 +319,7 @@ void emit_arithmetic_check(codegendata *cd, instruction *iptr, s4 reg)
 	if (INSTRUCTION_MUST_CHECK(iptr)) {
 		M_TEST(reg);
 		M_BNE(8);
-		M_ALD_MEM(reg, EXCEPTION_HARDWARE_ARITHMETIC);
+		M_ALD_MEM(reg, TRAP_ArithmeticException);
 	}
 }
 
@@ -354,7 +336,7 @@ void emit_arrayindexoutofbounds_check(codegendata *cd, instruction *iptr, s4 s1,
         M_ILD(REG_ITMP3, s1, OFFSET(java_array_t, size));
         M_ICMP(REG_ITMP3, s2);
 		M_BULT(8);
-		M_ALD_MEM(s2, EXCEPTION_HARDWARE_ARRAYINDEXOUTOFBOUNDS);
+		M_ALD_MEM(s2, TRAP_ArrayIndexOutOfBoundsException);
 	}
 }
 
@@ -370,7 +352,7 @@ void emit_arraystore_check(codegendata *cd, instruction *iptr)
 	if (INSTRUCTION_MUST_CHECK(iptr)) {
 		M_TEST(REG_RESULT);
 		M_BNE(8);
-		M_ALD_MEM(REG_RESULT, EXCEPTION_HARDWARE_ARRAYSTORE);
+		M_ALD_MEM(REG_RESULT, TRAP_ArrayStoreException);
 	}
 }
 
@@ -388,16 +370,22 @@ void emit_classcast_check(codegendata *cd, instruction *iptr, s4 condition, s4 r
 		case BRANCH_LE:
 			M_BGT(8);
 			break;
+		case BRANCH_GE:
+			M_BLT(8);
+			break;
 		case BRANCH_EQ:
 			M_BNE(8);
 			break;
+		case BRANCH_NE:
+			M_BEQ(8);
+			break;
 		case BRANCH_UGT:
 			M_BULE(8);
 			break;
 		default:
 			vm_abort("emit_classcast_check: unknown condition %d", condition);
 		}
-		M_ALD_MEM(s1, EXCEPTION_HARDWARE_CLASSCAST);
+		M_ALD_MEM(s1, TRAP_ClassCastException);
 	}
 }
 
@@ -413,7 +401,7 @@ void emit_nullpointer_check(codegendata *cd, instruction *iptr, s4 reg)
 	if (INSTRUCTION_MUST_CHECK(iptr)) {
 		M_TEST(reg);
 		M_BNE(8);
-		M_ALD_MEM(reg, EXCEPTION_HARDWARE_NULLPOINTER);
+		M_ALD_MEM(reg, TRAP_NullPointerException);
 	}
 }
 
@@ -429,117 +417,259 @@ void emit_exception_check(codegendata *cd, instruction *iptr)
 	if (INSTRUCTION_MUST_CHECK(iptr)) {
 		M_TEST(REG_RESULT);
 		M_BNE(8);
-		M_ALD_MEM(REG_RESULT, EXCEPTION_HARDWARE_EXCEPTION);
+		M_ALD_MEM(REG_RESULT, TRAP_CHECK_EXCEPTION);
 	}
 }
 
 
-/* emit_patcher_stubs **********************************************************
+/* emit_trap_compiler **********************************************************
 
-   Generates the code for the patcher stubs.
+   Emit a trap instruction which calls the JIT compiler.
 
 *******************************************************************************/
 
-void emit_patcher_stubs(jitdata *jd)
+void emit_trap_compiler(codegendata *cd)
 {
-	codegendata *cd;
-	patchref    *pref;
-	u8           mcode;
-	u1          *savedmcodeptr;
-	u1          *tmpmcodeptr;
-	s4           targetdisp;
-	s4           disp;
+	M_ALD_MEM(REG_METHODPTR, TRAP_COMPILER);
+}
 
-	/* get required compiler data */
 
-	cd = jd->cd;
+/* emit_patcher_alignment ******************************************************
+
+   Emit NOP to ensure placement at an even address.
+
+*******************************************************************************/
+
+void emit_patcher_alignment(codegendata *cd)
+{
+	if ((uintptr_t) cd->mcodeptr & 1)
+		M_NOP;
+}
+
+
+/* emit_trap *******************************************************************
+
+   Emit a trap instruction and return the original machine code.
+
+*******************************************************************************/
 
-	/* generate code patching stub call code */
+uint32_t emit_trap(codegendata *cd)
+{
+	uint16_t mcode;
 
-	targetdisp = 0;
+	/* Get machine code which is patched back in later. The trap is 2
+	   bytes long. */
 
-	for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
-		/* check size of code segment */
+	mcode = *((uint16_t *) cd->mcodeptr);
 
-		MCODECHECK(512);
+	/* XXX This needs to be change to INT3 when the debugging problems
+	   with gdb are resolved. */
 
-		/* Get machine code which is patched back in later. A
-		   `call rel32' is 5 bytes long (but read 8 bytes). */
+	M_UD2;
 
-		savedmcodeptr = cd->mcodebase + pref->branchpos;
-		mcode = *((u8 *) savedmcodeptr);
+	return mcode;
+}
 
-		/* patch in `call rel32' to call the following code */
 
-		tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr              */
-		cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position     */
+/**
+ * Generates fast-path code for the below builtin.
+ *   Function:  LOCK_monitor_enter
+ *   Signature: (Ljava/lang/Object;)V
+ *   Slow-path: bool lock_monitor_enter(java_handle_t*);
+ */
+void emit_fastpath_monitor_enter(jitdata* jd, instruction* iptr, int d)
+{
+	// Get required compiler data.
+	codegendata* cd = jd->cd;
 
-		M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
+	// XXX Currently the fast-path always fails. Implement me!
+	M_CLR(d);
+}
+
+
+/**
+ * Generates fast-path code for the below builtin.
+ *   Function:  LOCK_monitor_exit
+ *   Signature: (Ljava/lang/Object;)V
+ *   Slow-path: bool lock_monitor_exit(java_handle_t*);
+ */
+void emit_fastpath_monitor_exit(jitdata* jd, instruction* iptr, int d)
+{
+	// Get required compiler data.
+	codegendata* cd = jd->cd;
 
-		cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr       */
+	// XXX Currently the fast-path always fails. Implement me!
+	M_CLR(d);
+}
 
-		/* move pointer to java_objectheader onto stack */
 
+/**
+ * Generates synchronization code to enter a monitor.
+ */
 #if defined(ENABLE_THREADS)
-		/* create a virtual java_objectheader */
+void emit_monitor_enter(jitdata* jd, int32_t syncslot_offset)
+{
+	int32_t p;
+
+	// Get required compiler data.
+	methodinfo*  m  = jd->m;
+	codegendata* cd = jd->cd;
+
+# if !defined(NDEBUG)
+	if (JITDATA_HAS_FLAG_VERBOSECALL(jd)) {
+		M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
+
+		for (p = 0; p < INT_ARG_CNT; p++)
+			M_LST(abi_registers_integer_argument[p], REG_SP, p * 8);
+
+		for (p = 0; p < FLT_ARG_CNT; p++)
+			M_DST(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
+
+		syncslot_offset += (INT_ARG_CNT + FLT_ARG_CNT) * 8;
+	}
+# endif
+
+	/* decide which monitor enter function to call */
+
+	if (m->flags & ACC_STATIC) {
+		M_MOV_IMM(&m->clazz->object.header, REG_A0);
+	}
+	else {
+		M_TEST(REG_A0);
+		M_BNE(8);
+		M_ALD_MEM(REG_A0, TRAP_NullPointerException);
+	}
+
+	M_AST(REG_A0, REG_SP, syncslot_offset);
+	M_MOV_IMM(LOCK_monitor_enter, REG_ITMP1);
+	M_CALL(REG_ITMP1);
+
+# if !defined(NDEBUG)
+	if (JITDATA_HAS_FLAG_VERBOSECALL(jd)) {
 
-		(void) dseg_add_unique_address(cd, NULL);                  /* flcword */
-		(void) dseg_add_unique_address(cd, lock_get_initial_lock_word());
-		disp = dseg_add_unique_address(cd, NULL);                  /* vftbl   */
+		for (p = 0; p < INT_ARG_CNT; p++)
+			M_LLD(abi_registers_integer_argument[p], REG_SP, p * 8);
 
-		emit_lea_membase_reg(cd, RIP, -((cd->mcodeptr + 7) - cd->mcodebase) + disp, REG_ITMP3);
-		M_PUSH(REG_ITMP3);
-#else
-		M_PUSH_IMM(0);
+		for (p = 0; p < FLT_ARG_CNT; p++)
+			M_DLD(abi_registers_float_argument[p], REG_SP, (INT_ARG_CNT + p) * 8);
+
+		M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
+	}
+# endif
+}
 #endif
 
-		/* move machine code bytes and classinfo pointer into registers */
 
-		M_MOV_IMM(mcode, REG_ITMP3);
-		M_PUSH(REG_ITMP3);
+/**
+ * Generates synchronization code to leave a monitor.
+ */
+#if defined(ENABLE_THREADS)
+void emit_monitor_exit(jitdata* jd, int32_t syncslot_offset)
+{
+	// Get required compiler data.
+	methodinfo*  m  = jd->m;
+	codegendata* cd = jd->cd;
 
-		M_MOV_IMM(pref->ref, REG_ITMP3);
-		M_PUSH(REG_ITMP3);
+	M_ALD(REG_A0, REG_SP, syncslot_offset);
 
-		M_MOV_IMM(pref->disp, REG_ITMP3);
-		M_PUSH(REG_ITMP3);
+	/* we need to save the proper return value */
+
+	methoddesc* md = m->parseddesc;
+
+	switch (md->returntype.type) {
+	case TYPE_INT:
+	case TYPE_ADR:
+	case TYPE_LNG:
+		M_LST(REG_RESULT, REG_SP, syncslot_offset);
+		break;
+	case TYPE_FLT:
+	case TYPE_DBL:
+		M_DST(REG_FRESULT, REG_SP, syncslot_offset);
+		break;
+	}
 
-		M_MOV_IMM(pref->patcher, REG_ITMP3);
-		M_PUSH(REG_ITMP3);
+	M_MOV_IMM(LOCK_monitor_exit, REG_ITMP1);
+	M_CALL(REG_ITMP1);
 
-		if (targetdisp == 0) {
-			targetdisp = cd->mcodeptr - cd->mcodebase;
+	/* and now restore the proper return value */
 
-			M_MOV_IMM(asm_patcher_wrapper, REG_ITMP3);
-			M_JMP(REG_ITMP3);
-		}
-		else {
-			M_JMP_IMM((cd->mcodebase + targetdisp) -
-					  (cd->mcodeptr + PATCHER_CALL_SIZE));
-		}
+	switch (md->returntype.type) {
+	case TYPE_INT:
+	case TYPE_ADR:
+	case TYPE_LNG:
+		M_LLD(REG_RESULT, REG_SP, syncslot_offset);
+		break;
+	case TYPE_FLT:
+	case TYPE_DBL:
+		M_DLD(REG_FRESULT, REG_SP, syncslot_offset);
+		break;
 	}
 }
+#endif
 
 
-/* emit_trap *******************************************************************
+/**
+ * Emit profiling code for method frequency counting.
+ */
+#if defined(ENABLE_PROFILING)
+void emit_profile_method(codegendata* cd, codeinfo* code)
+{
+	M_MOV_IMM(code, REG_ITMP3);
+	M_IINC_MEMBASE(REG_ITMP3, OFFSET(codeinfo, frequency));
+}
+#endif
 
-   Emit a trap instruction and return the original machine code.
 
-*******************************************************************************/
+/**
+ * Emit profiling code for basicblock frequency counting.
+ */
+#if defined(ENABLE_PROFILING)
+void emit_profile_basicblock(codegendata* cd, codeinfo* code, basicblock* bptr)
+{
+	M_MOV_IMM(code->bbfrequency, REG_ITMP3);
+	M_IINC_MEMBASE(REG_ITMP3, bptr->nr * 4);
+}
+#endif
 
-uint32_t emit_trap(codegendata *cd)
+
+/**
+ * Emit profiling code to start CPU cycle counting.
+ */
+#if defined(ENABLE_PROFILING)
+void emit_profile_cycle_start(codegendata* cd, codeinfo* code)
 {
-	uint32_t mcode;
+	M_PUSH(RAX);
+	M_PUSH(RDX);
 
-	/* Get machine code which is patched back in later. The
-	   trap is 1 instruction word long. */
+	M_MOV_IMM(code, REG_ITMP3);
+	M_RDTSC;
+	M_ISUB_MEMBASE(RAX, REG_ITMP3, OFFSET(codeinfo, cycles));
+	M_ISBB_MEMBASE(RDX, REG_ITMP3, OFFSET(codeinfo, cycles) + 4);
 
-	mcode = *((uint32_t *) cd->mcodeptr);
+	M_POP(RDX);
+	M_POP(RAX);
+}
+#endif
 
-	M_NOP;
 
-	return mcode;
+/**
+ * Emit profiling code to stop CPU cycle counting.
+ */
+#if defined(ENABLE_PROFILING)
+void emit_profile_cycle_stop(codegendata* cd, codeinfo* code)
+{
+	M_PUSH(RAX);
+	M_PUSH(RDX);
+
+	M_MOV_IMM(code, REG_ITMP3);
+	M_RDTSC;
+	M_IADD_MEMBASE(RAX, REG_ITMP3, OFFSET(codeinfo, cycles));
+	M_IADC_MEMBASE(RDX, REG_ITMP3, OFFSET(codeinfo, cycles) + 4);
+
+	M_POP(RDX);
+	M_POP(RAX);
 }
+#endif
 
 
 /* emit_verbosecall_enter ******************************************************
@@ -552,16 +682,19 @@ uint32_t emit_trap(codegendata *cd)
 void emit_verbosecall_enter(jitdata *jd)
 {
 	methodinfo   *m;
+	codeinfo     *code;
 	codegendata  *cd;
 	registerdata *rd;
 	methoddesc   *md;
-	s4            i, j, k;
+	s4            stackframesize;
+	s4            i, s;
 
 	/* get required compiler data */
 
-	m  = jd->m;
-	cd = jd->cd;
-	rd = jd->rd;
+	m    = jd->m;
+	code = jd->code;
+	cd   = jd->cd;
+	rd   = jd->rd;
 
 	md = m->parseddesc;
 
@@ -569,69 +702,94 @@ void emit_verbosecall_enter(jitdata *jd)
 
 	M_NOP;
 
-	/* additional +1 is for 16-byte stack alignment */
+	/* keep 16-byte stack alignment */
+
+	stackframesize = md->paramcount + ARG_CNT + TMP_CNT;
+	ALIGN_2(stackframesize);
 
-	M_LSUB_IMM((ARG_CNT + TMP_CNT + 1 + 1) * 8, REG_SP);
+	M_LSUB_IMM(stackframesize * 8, REG_SP);
 
 	/* save argument registers */
 
-	for (i = 0; i < INT_ARG_CNT; i++)
-		M_LST(abi_registers_integer_argument[i], REG_SP, (1 + i) * 8);
+	for (i = 0; i < md->paramcount; i++) {
+		if (!md->params[i].inmemory) {
+			s = md->params[i].regoff;
+
+			switch (md->paramtypes[i].type) {
+			case TYPE_ADR:
+			case TYPE_INT:
+			case TYPE_LNG:
+				M_LST(s, REG_SP, i * 8);
+				break;
+			case TYPE_FLT:
+			case TYPE_DBL:
+				M_DST(s, REG_SP, i * 8);
+				break;
+			}
+		}
+	}
+
+	/* save all argument and temporary registers for leaf methods */
 
-	for (i = 0; i < FLT_ARG_CNT; i++)
-		M_DST(abi_registers_float_argument[i], REG_SP, (1 + INT_ARG_CNT + i) * 8);
+	if (code_is_leafmethod(code)) {
+		for (i = 0; i < INT_ARG_CNT; i++)
+			M_LST(abi_registers_integer_argument[i], REG_SP, (md->paramcount + i) * 8);
 
-	/* save temporary registers for leaf methods */
+		for (i = 0; i < FLT_ARG_CNT; i++)
+			M_DST(abi_registers_float_argument[i], REG_SP, (md->paramcount + INT_ARG_CNT + i) * 8);
 
-	if (jd->isleafmethod) {
 		for (i = 0; i < INT_TMP_CNT; i++)
-			M_LST(rd->tmpintregs[i], REG_SP, (1 + ARG_CNT + i) * 8);
+			M_LST(rd->tmpintregs[i], REG_SP, (md->paramcount + ARG_CNT + i) * 8);
 
 		for (i = 0; i < FLT_TMP_CNT; i++)
-			M_DST(rd->tmpfltregs[i], REG_SP, (1 + ARG_CNT + INT_TMP_CNT + i) * 8);
+			M_DST(rd->tmpfltregs[i], REG_SP, (md->paramcount + ARG_CNT + INT_TMP_CNT + i) * 8);
 	}
 
-	/* show integer hex code for float arguments */
+	M_MOV_IMM(m, REG_A0);
+	M_MOV(REG_SP, REG_A1);
+	M_MOV(REG_SP, REG_A2);
+	M_AADD_IMM((stackframesize + cd->stackframesize + 1) * 8, REG_A2);
+	M_MOV_IMM(trace_java_call_enter, REG_ITMP1);
+	M_CALL(REG_ITMP1);
+
+	/* restore argument registers */
 
-	for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++) {
-		/* If the paramtype is a float, we have to right shift all
-		   following integer registers. */
-	
-		if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
-			for (k = INT_ARG_CNT - 2; k >= i; k--)
-				M_MOV(abi_registers_integer_argument[k],
-					  abi_registers_integer_argument[k + 1]);
-
-			emit_movd_freg_reg(cd, abi_registers_float_argument[j],
-							   abi_registers_integer_argument[i]);
-			j++;
+	for (i = 0; i < md->paramcount; i++) {
+		if (!md->params[i].inmemory) {
+			s = md->params[i].regoff;
+
+			switch (md->paramtypes[i].type) {
+			case TYPE_ADR:
+			case TYPE_INT:
+			case TYPE_LNG:
+				M_LLD(s, REG_SP, i * 8);
+				break;
+			case TYPE_FLT:
+			case TYPE_DBL:
+				M_DLD(s, REG_SP, i * 8);
+				break;
+			}
 		}
 	}
 
-	M_MOV_IMM(m, REG_ITMP2);
-	M_AST(REG_ITMP2, REG_SP, 0 * 8);
-	M_MOV_IMM(builtin_verbosecall_enter, REG_ITMP1);
-	M_CALL(REG_ITMP1);
-
-	/* restore argument registers */
 
-	for (i = 0; i < INT_ARG_CNT; i++)
-		M_LLD(abi_registers_integer_argument[i], REG_SP, (1 + i) * 8);
+	/* restore all argument and temporary registers for leaf methods */
 
-	for (i = 0; i < FLT_ARG_CNT; i++)
-		M_DLD(abi_registers_float_argument[i], REG_SP, (1 + INT_ARG_CNT + i) * 8);
+	if (code_is_leafmethod(code)) {
+		for (i = 0; i < INT_ARG_CNT; i++)
+			M_LLD(abi_registers_integer_argument[i], REG_SP, (md->paramcount + i) * 8);
 
-	/* restore temporary registers for leaf methods */
+		for (i = 0; i < FLT_ARG_CNT; i++)
+			M_DLD(abi_registers_float_argument[i], REG_SP, (md->paramcount + INT_ARG_CNT + i) * 8);
 
-	if (jd->isleafmethod) {
 		for (i = 0; i < INT_TMP_CNT; i++)
-			M_LLD(rd->tmpintregs[i], REG_SP, (1 + ARG_CNT + i) * 8);
+			M_LLD(rd->tmpintregs[i], REG_SP, (md->paramcount + ARG_CNT + i) * 8);
 
 		for (i = 0; i < FLT_TMP_CNT; i++)
-			M_DLD(rd->tmpfltregs[i], REG_SP, (1 + ARG_CNT + INT_TMP_CNT + i) * 8);
+			M_DLD(rd->tmpfltregs[i], REG_SP, (md->paramcount + ARG_CNT + INT_TMP_CNT + i) * 8);
 	}
 
-	M_LADD_IMM((ARG_CNT + TMP_CNT + 1 + 1) * 8, REG_SP);
+	M_LADD_IMM(stackframesize * 8, REG_SP);
 
 	/* mark trace code */
 
@@ -652,6 +810,7 @@ void emit_verbosecall_exit(jitdata *jd)
 	methodinfo   *m;
 	codegendata  *cd;
 	registerdata *rd;
+	methoddesc   *md;
 
 	/* get required compiler data */
 
@@ -659,25 +818,49 @@ void emit_verbosecall_exit(jitdata *jd)
 	cd = jd->cd;
 	rd = jd->rd;
 
+	md = m->parseddesc;
+
 	/* mark trace code */
 
 	M_NOP;
 
+	/* keep 16-byte stack alignment */
+
 	M_ASUB_IMM(2 * 8, REG_SP);
 
-	M_LST(REG_RESULT, REG_SP, 0 * 8);
-	M_DST(REG_FRESULT, REG_SP, 1 * 8);
+	/* save return value */
+
+	switch (md->returntype.type) {
+	case TYPE_ADR:
+	case TYPE_INT:
+	case TYPE_LNG:
+		M_LST(REG_RESULT, REG_SP, 0 * 8);
+		break;
+	case TYPE_FLT:
+	case TYPE_DBL:
+		M_DST(REG_FRESULT, REG_SP, 0 * 8);
+		break;
+	}
 
-	M_INTMOVE(REG_RESULT, REG_A0);
-	M_FLTMOVE(REG_FRESULT, REG_FA0);
-	M_FLTMOVE(REG_FRESULT, REG_FA1);
-	M_MOV_IMM(m, REG_A1);
+	M_MOV_IMM(m, REG_A0);
+	M_MOV(REG_SP, REG_A1);
 
-	M_MOV_IMM(builtin_verbosecall_exit, REG_ITMP1);
+	M_MOV_IMM(trace_java_call_exit, REG_ITMP1);
 	M_CALL(REG_ITMP1);
 
-	M_LLD(REG_RESULT, REG_SP, 0 * 8);
-	M_DLD(REG_FRESULT, REG_SP, 1 * 8);
+	/* restore return value */
+
+	switch (md->returntype.type) {
+	case TYPE_ADR:
+	case TYPE_INT:
+	case TYPE_LNG:
+		M_LLD(REG_RESULT, REG_SP, 0 * 8);
+		break;
+	case TYPE_FLT:
+	case TYPE_DBL:
+		M_DLD(REG_FRESULT, REG_SP, 0 * 8);
+		break;
+	}
 
 	M_AADD_IMM(2 * 8, REG_SP);
 
@@ -1022,6 +1205,76 @@ void emit_lshift(jitdata *jd, s4 shift_op, instruction *iptr)
 
 /* low-level code emitter functions *******************************************/
 
+void emit_nop(codegendata *cd, int length)
+{
+    assert(length >= 1 && length <= 9);
+    switch (length) {
+    case 1:
+        *(cd->mcodeptr++) = 0x90;
+        break;
+    case 2:
+        *(cd->mcodeptr++) = 0x66;
+        *(cd->mcodeptr++) = 0x90;
+        break;
+    case 3:
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    case 4:
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x40;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    case 5:
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x44;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    case 6:
+        *(cd->mcodeptr++) = 0x66;
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x44;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    case 7:
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x80;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    case 8:
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x84;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    case 9:
+        *(cd->mcodeptr++) = 0x66;
+        *(cd->mcodeptr++) = 0x0f;
+        *(cd->mcodeptr++) = 0x1f;
+        *(cd->mcodeptr++) = 0x84;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        *(cd->mcodeptr++) = 0x00;
+        break;
+    }
+}
+
 void emit_mov_reg_reg(codegendata *cd, s8 reg, s8 dreg)
 {
 	emit_rex(1,(reg),0,(dreg));
@@ -1225,6 +1478,16 @@ void emit_movslq_reg_reg(codegendata *cd, s8 reg, s8 dreg)
 }
 
 
+void emit_movzbq_reg_reg(codegendata *cd, s8 reg, s8 dreg)
+{
+	emit_rex(1,(dreg),0,(reg));
+	*(cd->mcodeptr++) = 0x0f;
+	*(cd->mcodeptr++) = 0xb6;
+	/* XXX: why do reg and dreg have to be exchanged */
+	emit_reg((dreg),(reg));
+}
+
+
 void emit_movzwq_reg_reg(codegendata *cd, s8 reg, s8 dreg)
 {
 	emit_rex(1,(dreg),0,(reg));
@@ -1406,12 +1669,12 @@ void emit_alul_imm_reg(codegendata *cd, s8 opc, s8 imm, s8 dreg) {
 
 void emit_alu_imm_membase(codegendata *cd, s8 opc, s8 imm, s8 basereg, s8 disp) {
 	if (IS_IMM8(imm)) {
-		emit_rex(1,(basereg),0,0);
+		emit_rex(1,0,0,(basereg));
 		*(cd->mcodeptr++) = 0x83;
 		emit_membase(cd, (basereg),(disp),(opc));
 		emit_imm8((imm));
 	} else {
-		emit_rex(1,(basereg),0,0);
+		emit_rex(1,0,0,(basereg));
 		*(cd->mcodeptr++) = 0x81;
 		emit_membase(cd, (basereg),(disp),(opc));
 		emit_imm32((imm));
@@ -1421,18 +1684,31 @@ void emit_alu_imm_membase(codegendata *cd, s8 opc, s8 imm, s8 basereg, s8 disp)
 
 void emit_alul_imm_membase(codegendata *cd, s8 opc, s8 imm, s8 basereg, s8 disp) {
 	if (IS_IMM8(imm)) {
-		emit_rex(0,(basereg),0,0);
+		emit_rex(0,0,0,(basereg));
 		*(cd->mcodeptr++) = 0x83;
 		emit_membase(cd, (basereg),(disp),(opc));
 		emit_imm8((imm));
 	} else {
-		emit_rex(0,(basereg),0,0);
+		emit_rex(0,0,0,(basereg));
 		*(cd->mcodeptr++) = 0x81;
 		emit_membase(cd, (basereg),(disp),(opc));
 		emit_imm32((imm));
 	}
 }
 
+void emit_alu_memindex_reg(codegendata *cd, s8 opc, s8 disp, s8 basereg, s8 indexreg, s8 scale, s8 reg)
+{
+	emit_rex(1,(reg),(indexreg),(basereg));
+	*(cd->mcodeptr++) = (((opc)) << 3) + 3;
+	emit_memindex(cd, (reg),(disp),(basereg),(indexreg),(scale));
+}
+
+void emit_alul_memindex_reg(codegendata *cd, s8 opc, s8 disp, s8 basereg, s8 indexreg, s8 scale, s8 reg)
+{
+	emit_rex(0,(reg),(indexreg),(basereg));
+	*(cd->mcodeptr++) = (((opc)) << 3) + 3;
+	emit_memindex(cd, (reg),(disp),(basereg),(indexreg),(scale));
+}
 
 void emit_test_reg_reg(codegendata *cd, s8 reg, s8 dreg) {
 	emit_rex(1,(reg),0,(dreg));
@@ -1484,6 +1760,18 @@ void emit_leal_membase_reg(codegendata *cd, s8 basereg, s8 disp, s8 reg) {
 }
 
 
+void emit_incl_reg(codegendata *cd, s8 reg)
+{
+	*(cd->mcodeptr++) = 0xff;
+	emit_reg(0,(reg));
+}
+
+void emit_incq_reg(codegendata *cd, s8 reg)
+{
+	emit_rex(1,0,0,(reg));
+	*(cd->mcodeptr++) = 0xff;
+	emit_reg(0,(reg));
+}
 
 void emit_incl_membase(codegendata *cd, s8 basereg, s8 disp)
 {
@@ -1492,6 +1780,13 @@ void emit_incl_membase(codegendata *cd, s8 basereg, s8 disp)
 	emit_membase(cd, (basereg),(disp),0);
 }
 
+void emit_incq_membase(codegendata *cd, s8 basereg, s8 disp)
+{
+	emit_rex(1,0,0,(basereg));
+	*(cd->mcodeptr++) = 0xff;
+	emit_membase(cd, (basereg),(disp),0);
+}
+
 
 
 void emit_cltd(codegendata *cd) {
@@ -1724,6 +2019,18 @@ void emit_jmp_imm(codegendata *cd, s8 imm) {
 	emit_imm32((imm));
 }
 
+/* like emit_jmp_imm but allows 8 bit optimization */
+void emit_jmp_imm2(codegendata *cd, s8 imm) {
+	if (IS_IMM8(imm)) {
+		*(cd->mcodeptr++) = 0xeb;
+		emit_imm8((imm));
+	}
+	else {
+		*(cd->mcodeptr++) = 0xe9;
+		emit_imm32((imm));
+	}
+}
+
 
 void emit_jmp_reg(codegendata *cd, s8 reg) {
 	emit_rex(0,0,0,(reg));
@@ -2336,6 +2643,13 @@ void emit_rdtsc(codegendata *cd)
 	*(cd->mcodeptr++) = 0x31;
 }
 
+void emit_mfence(codegendata *cd)
+{
+	*(cd->mcodeptr++) = 0x0f;
+	*(cd->mcodeptr++) = 0xae;
+	*(cd->mcodeptr++) = 0xf0;
+}
+
 
 /*
  * These are local overrides for various environment variables in Emacs.
@@ -2348,4 +2662,5 @@ void emit_rdtsc(codegendata *cd)
  * c-basic-offset: 4
  * tab-width: 4
  * End:
+ * vim:noexpandtab:sw=4:ts=4:
  */