src/vm/jit/x86_64/codegen.c

   1 /* jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    $Id: codegen.c 1735 2004-12-07 14:33:27Z twisti $
  31
  32 */
  33
  34
  35 #define _GNU_SOURCE
  36
  37 #include <stdio.h>
  38 #include <ucontext.h>
  39
  40 #include "native/native.h"
  41 /*  #include "native/jni.h" */
  42 #include "vm/global.h"
  43 #include "vm/builtin.h"
  44 #include "vm/loader.h"
  45 #include "vm/tables.h"
  46 #include "vm/jit/asmpart.h"
  47 #include "vm/jit/jit.h"
  48 #include "vm/jit/reg.h"
  49 #include "vm/jit/parse.h"
  50 #include "vm/jit/x86_64/arch.h"
  51 #include "vm/jit/x86_64/codegen.h"
  52 #include "vm/jit/x86_64/emitfuncs.h"
  53 #include "vm/jit/x86_64/types.h"
  54 #include "vm/jit/x86_64/asmoffsets.h"
  55
  56
  57 /* register descripton - array ************************************************/
  58
  59 /* #define REG_RES   0         reserved register for OS or code generator     */
  60 /* #define REG_RET   1         return value register                          */
  61 /* #define REG_EXC   2         exception value register (only old jit)        */
  62 /* #define REG_SAV   3         (callee) saved register                        */
  63 /* #define REG_TMP   4         scratch temporary register (caller saved)      */
  64 /* #define REG_ARG   5         argument register (caller saved)               */
  65
  66 /* #define REG_END   -1        last entry in tables                           */
  67
  68 static int nregdescint[] = {
  69     REG_RET, REG_ARG, REG_ARG, REG_TMP, REG_RES, REG_SAV, REG_ARG, REG_ARG,
  70     REG_ARG, REG_ARG, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV,
  71     REG_END
  72 };
  73
  74
  75 static int nregdescfloat[] = {
  76 /*      REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP, */
  77 /*      REG_RES, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV, REG_SAV, */
  78     REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  79     REG_RES, REG_RES, REG_RES, REG_TMP, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  80     REG_END
  81 };
  82
  83
  84 /* Include independent code generation stuff -- include after register        */
  85 /* descriptions to avoid extern definitions.                                  */
  86
  87 #include "vm/jit/codegen.inc"
  88 #include "vm/jit/reg.inc"
  89 #ifdef LSRA
  90 #include "vm/jit/lsra.inc"
  91 #endif
  92
  93
  94 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
  95 void thread_restartcriticalsection(ucontext_t *uc)
  96 {
  97         void *critical;
  98
  99         critical = thread_checkcritical((void *) uc->uc_mcontext.gregs[REG_RIP]);
 100
 101         if (critical)
 102                 uc->uc_mcontext.gregs[REG_RIP] = (u8) critical;
 103 }
 104 #endif
 105
 106
 107 /* NullPointerException signal handler for hardware null pointer check */
 108
 109 void catch_NullPointerException(int sig, siginfo_t *siginfo, void *_p)
 110 {
 111         sigset_t nsig;
 112         /*      int      instr; */
 113         /*      long     faultaddr; */
 114
 115         struct ucontext *_uc = (struct ucontext *) _p;
 116         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 117         struct sigaction act;
 118         java_objectheader *xptr;
 119
 120         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 121
 122
 123 /*      instr = *((int*)(sigctx->rip)); */
 124 /*      faultaddr = sigctx->sc_regs[(instr >> 16) & 0x1f]; */
 125
 126 /*      if (faultaddr == 0) { */
 127         act.sa_sigaction = (functionptr) catch_NullPointerException; /* reinstall handler */
 128         act.sa_flags = SA_SIGINFO;
 129         sigaction(sig, &act, NULL);
 130
 131         sigemptyset(&nsig);
 132         sigaddset(&nsig, sig);
 133         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 134
 135         xptr = new_nullpointerexception();
 136
 137         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 138         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 139         sigctx->rip = (u8) asm_handle_exception;
 140
 141         return;
 142
 143 /*      } else { */
 144 /*              faultaddr += (long) ((instr << 16) >> 16); */
 145 /*              fprintf(stderr, "faulting address: 0x%08x\n", faultaddr); */
 146 /*              panic("Stack overflow"); */
 147 /*      } */
 148 }
 149
 150
 151 /* ArithmeticException signal handler for hardware divide by zero check */
 152
 153 void catch_ArithmeticException(int sig, siginfo_t *siginfo, void *_p)
 154 {
 155         sigset_t nsig;
 156
 157         struct ucontext *_uc = (struct ucontext *) _p;
 158         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 159         struct sigaction act;
 160         java_objectheader *xptr;
 161
 162         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 163
 164         act.sa_sigaction = (functionptr) catch_ArithmeticException; /* reinstall handler */
 165         act.sa_flags = SA_SIGINFO;
 166         sigaction(sig, &act, NULL);
 167
 168         sigemptyset(&nsig);
 169         sigaddset(&nsig, sig);
 170         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 171
 172         xptr = new_arithmeticexception();
 173
 174         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 175         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 176         sigctx->rip = (u8) asm_handle_exception;
 177
 178         return;
 179 }
 180
 181
 182 void init_exceptions(void)
 183 {
 184         struct sigaction act;
 185
 186         /* install signal handlers we need to convert to exceptions */
 187         sigemptyset(&act.sa_mask);
 188
 189         if (!checknull) {
 190 #if defined(SIGSEGV)
 191                 act.sa_sigaction = (functionptr) catch_NullPointerException;
 192                 act.sa_flags = SA_SIGINFO;
 193                 sigaction(SIGSEGV, &act, NULL);
 194 #endif
 195
 196 #if defined(SIGBUS)
 197                 act.sa_sigaction = (functionptr) catch_NullPointerException;
 198                 act.sa_flags = SA_SIGINFO;
 199                 sigaction(SIGBUS, &act, NULL);
 200 #endif
 201         }
 202
 203         act.sa_sigaction = (functionptr) catch_ArithmeticException;
 204         act.sa_flags = SA_SIGINFO;
 205         sigaction(SIGFPE, &act, NULL);
 206 }
 207
 208
 209 /* function gen_mcode **********************************************************
 210
 211         generates machine code
 212
 213 *******************************************************************************/
 214
 215 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
 216 {
 217         s4 len, s1, s2, s3, d;
 218         s8 a;
 219         s4 parentargs_base;
 220         stackptr        src;
 221         varinfo        *var;
 222         basicblock     *bptr;
 223         instruction    *iptr;
 224         exceptiontable *ex;
 225
 226         {
 227         s4 i, p, pa, t, l;
 228         s4 savedregs_num;
 229
 230         savedregs_num = 0;
 231
 232         /* space to save used callee saved registers */
 233
 234         savedregs_num += (rd->savintregcnt - rd->maxsavintreguse);
 235         savedregs_num += (rd->savfltregcnt - rd->maxsavfltreguse);
 236
 237         parentargs_base = rd->maxmemuse + savedregs_num;
 238
 239 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 240
 241         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 242                 parentargs_base++;
 243
 244 #endif
 245
 246     /* keep stack 16-byte aligned for calls into native code e.g. libc or jni */
 247     /* (alignment problems with movaps)                                       */
 248
 249         if (!(parentargs_base & 0x1)) {
 250                 parentargs_base++;
 251         }
 252
 253         /* create method header */
 254
 255         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 256         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 257
 258 #if defined(USE_THREADS)
 259
 260         /* IsSync contains the offset relative to the stack pointer for the
 261            argument of monitor_exit used in the exception handler. Since the
 262            offset could be zero and give a wrong meaning of the flag it is
 263            offset by one.
 264         */
 265
 266         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 267                 (void) dseg_adds4(cd, (rd->maxmemuse + 1) * 8);     /* IsSync         */
 268         else
 269
 270 #endif
 271
 272                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 273
 274         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 275         (void) dseg_adds4(cd, rd->savintregcnt - rd->maxsavintreguse);/* IntSave  */
 276         (void) dseg_adds4(cd, rd->savfltregcnt - rd->maxsavfltreguse);/* FltSave  */
 277         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 278
 279         /* create exception table */
 280
 281         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 282                 dseg_addtarget(cd, ex->start);
 283                 dseg_addtarget(cd, ex->end);
 284                 dseg_addtarget(cd, ex->handler);
 285                 (void) dseg_addaddress(cd, ex->catchtype);
 286         }
 287
 288         /* initialize mcode variables */
 289
 290         cd->mcodeptr = (u1 *) cd->mcodebase;
 291         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 292         MCODECHECK(128 + m->paramcount);
 293
 294         /* create stack frame (if necessary) */
 295
 296         if (parentargs_base) {
 297                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 298         }
 299
 300         /* save return address and used callee saved registers */
 301
 302         p = parentargs_base;
 303         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
 304                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 305         }
 306         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
 307                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 308         }
 309
 310         /* save monitorenter argument */
 311
 312 #if defined(USE_THREADS)
 313         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 314                 if (m->flags & ACC_STATIC) {
 315                         x86_64_mov_imm_reg(cd, (s8) m->class, REG_ITMP1);
 316                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, rd->maxmemuse * 8);
 317
 318                 } else {
 319                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, rd->maxmemuse * 8);
 320                 }
 321         }
 322 #endif
 323
 324         /* copy argument registers to stack and call trace function with pointer
 325            to arguments on stack.
 326         */
 327         if (runverbose) {
 328                 x86_64_alu_imm_reg(cd, X86_64_SUB, (6 + 8 + 1 + 1) * 8, REG_SP);
 329
 330                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
 331                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
 332                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
 333                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
 334                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
 335                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
 336
 337                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
 338                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
 339                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
 340                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
 341 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8); */
 342 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8); */
 343 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8); */
 344 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8); */
 345
 346                 for (p = 0, l = 0; p < m->paramcount; p++) {
 347                         t = m->paramtypes[p];
 348
 349                         if (IS_FLT_DBL_TYPE(t)) {
 350                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
 351                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
 352                                 }
 353
 354                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 355                                 l++;
 356                         }
 357                 }
 358
 359                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP2);
 360                 x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 361                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
 362                 x86_64_call_reg(cd, REG_ITMP1);
 363
 364                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
 365                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
 366                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
 367                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
 368                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
 369                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
 370
 371                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
 372                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
 373                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
 374                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
 375 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]); */
 376 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]); */
 377 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]); */
 378 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]); */
 379
 380                 x86_64_alu_imm_reg(cd, X86_64_ADD, (6 + 8 + 1 + 1) * 8, REG_SP);
 381         }
 382
 383         /* take arguments out of register or stack frame */
 384
 385         for (p = 0, l = 0, s1 = 0, s2 = 0; p < m->paramcount; p++) {
 386                 t = m->paramtypes[p];
 387                 var = &(rd->locals[l][t]);
 388                 l++;
 389                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 390                         l++;
 391                 if (var->type < 0) {
 392                         if (IS_INT_LNG_TYPE(t)) {
 393                                 s1++;
 394                         } else {
 395                                 s2++;
 396                         }
 397                         continue;
 398                 }
 399                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 400                         if (s1 < INT_ARG_CNT) {                  /* register arguments    */
 401                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 402                                         M_INTMOVE(rd->argintregs[s1], var->regoff);
 403
 404                                 } else {                             /* reg arg -> spilled    */
 405                                     x86_64_mov_reg_membase(cd, rd->argintregs[s1], REG_SP, var->regoff * 8);
 406                                 }
 407
 408                         } else {                                 /* stack arguments       */
 409                                 pa = s1 - INT_ARG_CNT;
 410                                 if (s2 >= FLT_ARG_CNT) {
 411                                         pa += s2 - FLT_ARG_CNT;
 412                                 }
 413                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 414                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);    /* + 8 for return address */
 415                                 } else {                             /* stack arg -> spilled  */
 416                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_ITMP1);    /* + 8 for return address */
 417                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, var->regoff * 8);
 418                                 }
 419                         }
 420                         s1++;
 421
 422                 } else {                                     /* floating args         */
 423                         if (s2 < FLT_ARG_CNT) {                /* register arguments    */
 424                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 425                                         M_FLTMOVE(rd->argfltregs[s2], var->regoff);
 426
 427                                 } else {                                         /* reg arg -> spilled    */
 428                                         x86_64_movq_reg_membase(cd, rd->argfltregs[s2], REG_SP, var->regoff * 8);
 429                                 }
 430
 431                         } else {                                 /* stack arguments       */
 432                                 pa = s2 - FLT_ARG_CNT;
 433                                 if (s1 >= INT_ARG_CNT) {
 434                                         pa += s1 - INT_ARG_CNT;
 435                                 }
 436                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 437                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);
 438
 439                                 } else {
 440                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_FTMP1);
 441                                         x86_64_movq_reg_membase(cd, REG_FTMP1, REG_SP, var->regoff * 8);
 442                                 }
 443                         }
 444                         s2++;
 445                 }
 446         }  /* end for */
 447
 448         /* call monitorenter function */
 449
 450 #if defined(USE_THREADS)
 451         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 452                 s8 func_enter = (m->flags & ACC_STATIC) ?
 453                         (s8) builtin_staticmonitorenter : (s8) builtin_monitorenter;
 454                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
 455                 x86_64_mov_imm_reg(cd, func_enter, REG_ITMP1);
 456                 x86_64_call_reg(cd, REG_ITMP1);
 457         }
 458 #endif
 459         }
 460
 461         /* end of header generation */
 462
 463         /* walk through all basic blocks */
 464         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 465
 466                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 467
 468                 if (bptr->flags >= BBREACHED) {
 469
 470                         /* branch resolving */
 471
 472                         branchref *bref;
 473                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 474                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 475                                                                   bref->branchpos,
 476                                                                   bptr->mpc);
 477                         }
 478
 479                 /* copy interface registers to their destination */
 480
 481                 src = bptr->instack;
 482                 len = bptr->indepth;
 483                 MCODECHECK(64 + len);
 484                 while (src != NULL) {
 485                         len--;
 486                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 487                                 if (bptr->type == BBTYPE_SBR) {
 488                                         d = reg_of_var(rd, src, REG_ITMP1);
 489                                         x86_64_pop_reg(cd, d);
 490                                         store_reg_to_var_int(src, d);
 491
 492                                 } else if (bptr->type == BBTYPE_EXH) {
 493                                         d = reg_of_var(rd, src, REG_ITMP1);
 494                                         M_INTMOVE(REG_ITMP1, d);
 495                                         store_reg_to_var_int(src, d);
 496                                 }
 497
 498                         } else {
 499                                 d = reg_of_var(rd, src, REG_ITMP1);
 500                                 if ((src->varkind != STACKVAR)) {
 501                                         s2 = src->type;
 502                                         if (IS_FLT_DBL_TYPE(s2)) {
 503                                                 s1 = rd->interfaces[len][s2].regoff;
 504                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 505                                                         M_FLTMOVE(s1, d);
 506
 507                                                 } else {
 508                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 509                                                 }
 510                                                 store_reg_to_var_flt(src, d);
 511
 512                                         } else {
 513                                                 s1 = rd->interfaces[len][s2].regoff;
 514                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 515                                                         M_INTMOVE(s1, d);
 516
 517                                                 } else {
 518                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 519                                                 }
 520                                                 store_reg_to_var_int(src, d);
 521                                         }
 522                                 }
 523                         }
 524                         src = src->prev;
 525                 }
 526
 527                 /* walk through all instructions */
 528
 529                 src = bptr->instack;
 530                 len = bptr->icount;
 531                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 532
 533                         MCODECHECK(64);   /* an instruction usually needs < 64 words      */
 534                         switch (iptr->opc) {
 535
 536                         case ICMD_NOP:    /* ...  ==> ...                                 */
 537                                 break;
 538
 539                         case ICMD_NULLCHECKPOP: /* ..., objectref  ==> ...                */
 540                                 if (src->flags & INMEMORY) {
 541                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 542
 543                                 } else {
 544                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 545                                 }
 546                                 x86_64_jcc(cd, X86_64_CC_E, 0);
 547                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 548                                 break;
 549
 550                 /* constant operations ************************************************/
 551
 552                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 553                                       /* op1 = 0, val.i = constant                    */
 554
 555                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 556                         if (iptr->val.i == 0) {
 557                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 558                         } else {
 559                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 560                         }
 561                         store_reg_to_var_int(iptr->dst, d);
 562                         break;
 563
 564                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 565                                       /* op1 = 0, val.a = constant                    */
 566
 567                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 568                         if (iptr->val.a == 0) {
 569                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 570                         } else {
 571                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 572                         }
 573                         store_reg_to_var_int(iptr->dst, d);
 574                         break;
 575
 576                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 577                                       /* op1 = 0, val.l = constant                    */
 578
 579                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 580                         if (iptr->val.l == 0) {
 581                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 582                         } else {
 583                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 584                         }
 585                         store_reg_to_var_int(iptr->dst, d);
 586                         break;
 587
 588                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 589                                       /* op1 = 0, val.f = constant                    */
 590
 591                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 592                         a = dseg_addfloat(cd, iptr->val.f);
 593                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 594                         store_reg_to_var_flt(iptr->dst, d);
 595                         break;
 596
 597                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 598                                       /* op1 = 0, val.d = constant                    */
 599
 600                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 601                         a = dseg_adddouble(cd, iptr->val.d);
 602                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 603                         store_reg_to_var_flt(iptr->dst, d);
 604                         break;
 605
 606
 607                 /* load/store operations **********************************************/
 608
 609                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 610                                       /* op1 = local variable                         */
 611
 612                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 613                         if ((iptr->dst->varkind == LOCALVAR) &&
 614                             (iptr->dst->varnum == iptr->op1)) {
 615                                 break;
 616                         }
 617                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 618                         if (var->flags & INMEMORY) {
 619                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 620                                 store_reg_to_var_int(iptr->dst, d);
 621
 622                         } else {
 623                                 if (iptr->dst->flags & INMEMORY) {
 624                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 625
 626                                 } else {
 627                                         M_INTMOVE(var->regoff, d);
 628                                 }
 629                         }
 630                         break;
 631
 632                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 633                 case ICMD_ALOAD:      /* op1 = local variable                         */
 634
 635                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 636                         if ((iptr->dst->varkind == LOCALVAR) &&
 637                             (iptr->dst->varnum == iptr->op1)) {
 638                                 break;
 639                         }
 640                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 641                         if (var->flags & INMEMORY) {
 642                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 643                                 store_reg_to_var_int(iptr->dst, d);
 644
 645                         } else {
 646                                 if (iptr->dst->flags & INMEMORY) {
 647                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 648
 649                                 } else {
 650                                         M_INTMOVE(var->regoff, d);
 651                                 }
 652                         }
 653                         break;
 654
 655                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 656                 case ICMD_DLOAD:      /* op1 = local variable                         */
 657
 658                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 659                         if ((iptr->dst->varkind == LOCALVAR) &&
 660                             (iptr->dst->varnum == iptr->op1)) {
 661                                 break;
 662                         }
 663                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 664                         if (var->flags & INMEMORY) {
 665                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 666                                 store_reg_to_var_flt(iptr->dst, d);
 667
 668                         } else {
 669                                 if (iptr->dst->flags & INMEMORY) {
 670                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 671
 672                                 } else {
 673                                         M_FLTMOVE(var->regoff, d);
 674                                 }
 675                         }
 676                         break;
 677
 678                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 679                 case ICMD_LSTORE:     /* op1 = local variable                         */
 680                 case ICMD_ASTORE:
 681
 682                         if ((src->varkind == LOCALVAR) &&
 683                             (src->varnum == iptr->op1)) {
 684                                 break;
 685                         }
 686                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 687                         if (var->flags & INMEMORY) {
 688                                 var_to_reg_int(s1, src, REG_ITMP1);
 689                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 690
 691                         } else {
 692                                 var_to_reg_int(s1, src, var->regoff);
 693                                 M_INTMOVE(s1, var->regoff);
 694                         }
 695                         break;
 696
 697                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 698                 case ICMD_DSTORE:     /* op1 = local variable                         */
 699
 700                         if ((src->varkind == LOCALVAR) &&
 701                             (src->varnum == iptr->op1)) {
 702                                 break;
 703                         }
 704                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 705                         if (var->flags & INMEMORY) {
 706                                 var_to_reg_flt(s1, src, REG_FTMP1);
 707                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 708
 709                         } else {
 710                                 var_to_reg_flt(s1, src, var->regoff);
 711                                 M_FLTMOVE(s1, var->regoff);
 712                         }
 713                         break;
 714
 715
 716                 /* pop/dup/swap operations ********************************************/
 717
 718                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 719
 720                 case ICMD_POP:        /* ..., value  ==> ...                          */
 721                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 722                         break;
 723
 724                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 725                         M_COPY(src, iptr->dst);
 726                         break;
 727
 728                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 729
 730                         M_COPY(src,       iptr->dst);
 731                         M_COPY(src->prev, iptr->dst->prev);
 732                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 733                         break;
 734
 735                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 736
 737                         M_COPY(src,             iptr->dst);
 738                         M_COPY(src->prev,       iptr->dst->prev);
 739                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 740                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 741                         break;
 742
 743                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 744
 745                         M_COPY(src,       iptr->dst);
 746                         M_COPY(src->prev, iptr->dst->prev);
 747                         break;
 748
 749                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 750
 751                         M_COPY(src,             iptr->dst);
 752                         M_COPY(src->prev,       iptr->dst->prev);
 753                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 754                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 755                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 756                         break;
 757
 758                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 759
 760                         M_COPY(src,                   iptr->dst);
 761                         M_COPY(src->prev,             iptr->dst->prev);
 762                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 763                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 764                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 765                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 766                         break;
 767
 768                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 769
 770                         M_COPY(src,       iptr->dst->prev);
 771                         M_COPY(src->prev, iptr->dst);
 772                         break;
 773
 774
 775                 /* integer operations *************************************************/
 776
 777                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 778
 779                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 780                         if (iptr->dst->flags & INMEMORY) {
 781                                 if (src->flags & INMEMORY) {
 782                                         if (src->regoff == iptr->dst->regoff) {
 783                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 784
 785                                         } else {
 786                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 787                                                 x86_64_negl_reg(cd, REG_ITMP1);
 788                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 789                                         }
 790
 791                                 } else {
 792                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 793                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 794                                 }
 795
 796                         } else {
 797                                 if (src->flags & INMEMORY) {
 798                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 799                                         x86_64_negl_reg(cd, d);
 800
 801                                 } else {
 802                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 803                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 804                                 }
 805                         }
 806                         break;
 807
 808                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 809
 810                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 811                         if (iptr->dst->flags & INMEMORY) {
 812                                 if (src->flags & INMEMORY) {
 813                                         if (src->regoff == iptr->dst->regoff) {
 814                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 815
 816                                         } else {
 817                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 818                                                 x86_64_neg_reg(cd, REG_ITMP1);
 819                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 820                                         }
 821
 822                                 } else {
 823                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 824                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 825                                 }
 826
 827                         } else {
 828                                 if (src->flags & INMEMORY) {
 829                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 830                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 831
 832                                 } else {
 833                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 834                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 835                                 }
 836                         }
 837                         break;
 838
 839                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 840
 841                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 842                         if (src->flags & INMEMORY) {
 843                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 844
 845                         } else {
 846                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 847                         }
 848                         store_reg_to_var_int(iptr->dst, d);
 849                         break;
 850
 851                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 852
 853                         var_to_reg_int(s1, src, REG_ITMP1);
 854                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 855                         M_INTMOVE(s1, d);
 856                         store_reg_to_var_int(iptr->dst, d);
 857                         break;
 858
 859                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 860
 861                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 862                         if (src->flags & INMEMORY) {
 863                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 864
 865                         } else {
 866                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 867                         }
 868                         store_reg_to_var_int(iptr->dst, d);
 869                         break;
 870
 871                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 872
 873                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 874                         if (src->flags & INMEMORY) {
 875                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 876
 877                         } else {
 878                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 879                         }
 880                         store_reg_to_var_int(iptr->dst, d);
 881                         break;
 882
 883                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 884
 885                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 886                         if (src->flags & INMEMORY) {
 887                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 888
 889                         } else {
 890                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 891                         }
 892                         store_reg_to_var_int(iptr->dst, d);
 893                         break;
 894
 895
 896                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 897
 898                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 899                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 900                         break;
 901
 902                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 903                                       /* val.i = constant                             */
 904
 905                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 906                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 907                         break;
 908
 909                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 910
 911                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 912                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 913                         break;
 914
 915                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 916                                       /* val.l = constant                             */
 917
 918                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 919                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 920                         break;
 921
 922                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 923
 924                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 925                         if (iptr->dst->flags & INMEMORY) {
 926                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 927                                         if (src->prev->regoff == iptr->dst->regoff) {
 928                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 929                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 930
 931                                         } else {
 932                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 933                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 934                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 935                                         }
 936
 937                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 938                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 939                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 940                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 941
 942                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 943                                         if (src->prev->regoff == iptr->dst->regoff) {
 944                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 945
 946                                         } else {
 947                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 948                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 949                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 950                                         }
 951
 952                                 } else {
 953                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 954                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 955                                 }
 956
 957                         } else {
 958                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 959                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 960                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 961
 962                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 963                                         M_INTMOVE(src->prev->regoff, d);
 964                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 965
 966                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 967                                         /* workaround for reg alloc */
 968                                         if (src->regoff == iptr->dst->regoff) {
 969                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 970                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 971                                                 M_INTMOVE(REG_ITMP1, d);
 972
 973                                         } else {
 974                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 975                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 976                                         }
 977
 978                                 } else {
 979                                         /* workaround for reg alloc */
 980                                         if (src->regoff == iptr->dst->regoff) {
 981                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 982                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 983                                                 M_INTMOVE(REG_ITMP1, d);
 984
 985                                         } else {
 986                                                 M_INTMOVE(src->prev->regoff, d);
 987                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 988                                         }
 989                                 }
 990                         }
 991                         break;
 992
 993                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 994                                       /* val.i = constant                             */
 995
 996                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 997                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 998                         break;
 999
1000                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1001
1002                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1003                         if (iptr->dst->flags & INMEMORY) {
1004                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1005                                         if (src->prev->regoff == iptr->dst->regoff) {
1006                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1007                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1008
1009                                         } else {
1010                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1011                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1012                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1013                                         }
1014
1015                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1016                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1017                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1018                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1019
1020                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1021                                         if (src->prev->regoff == iptr->dst->regoff) {
1022                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1023
1024                                         } else {
1025                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1026                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1027                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1028                                         }
1029
1030                                 } else {
1031                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1032                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1033                                 }
1034
1035                         } else {
1036                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1037                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1038                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1039
1040                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1041                                         M_INTMOVE(src->prev->regoff, d);
1042                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1043
1044                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1045                                         /* workaround for reg alloc */
1046                                         if (src->regoff == iptr->dst->regoff) {
1047                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1048                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1049                                                 M_INTMOVE(REG_ITMP1, d);
1050
1051                                         } else {
1052                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1053                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1054                                         }
1055
1056                                 } else {
1057                                         /* workaround for reg alloc */
1058                                         if (src->regoff == iptr->dst->regoff) {
1059                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1060                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1061                                                 M_INTMOVE(REG_ITMP1, d);
1062
1063                                         } else {
1064                                                 M_INTMOVE(src->prev->regoff, d);
1065                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1066                                         }
1067                                 }
1068                         }
1069                         break;
1070
1071                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1072                                       /* val.l = constant                             */
1073
1074                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1075                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1076                         break;
1077
1078                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1079
1080                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1081                         if (iptr->dst->flags & INMEMORY) {
1082                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1083                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1084                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1085                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1086
1087                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1088                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1089                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1090                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1091
1092                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1093                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1094                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1095                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1096
1097                                 } else {
1098                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1099                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1100                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1101                                 }
1102
1103                         } else {
1104                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1105                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1106                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1107
1108                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1109                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1110                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1111
1112                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1113                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1114                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1115
1116                                 } else {
1117                                         if (src->regoff == iptr->dst->regoff) {
1118                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1119
1120                                         } else {
1121                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1122                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1123                                         }
1124                                 }
1125                         }
1126                         break;
1127
1128                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1129                                       /* val.i = constant                             */
1130
1131                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1132                         if (iptr->dst->flags & INMEMORY) {
1133                                 if (src->flags & INMEMORY) {
1134                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1135                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1136
1137                                 } else {
1138                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1139                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1140                                 }
1141
1142                         } else {
1143                                 if (src->flags & INMEMORY) {
1144                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1145
1146                                 } else {
1147                                         if (iptr->val.i == 2) {
1148                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1149                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1150
1151                                         } else {
1152                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1153                                         }
1154                                 }
1155                         }
1156                         break;
1157
1158                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1159
1160                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1161                         if (iptr->dst->flags & INMEMORY) {
1162                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1163                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1164                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1165                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1166
1167                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1168                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1169                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1170                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1171
1172                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1173                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1174                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1175                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1176
1177                                 } else {
1178                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1179                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1180                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1181                                 }
1182
1183                         } else {
1184                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1185                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1186                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1187
1188                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1189                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1190                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1191
1192                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1193                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1194                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1195
1196                                 } else {
1197                                         if (src->regoff == iptr->dst->regoff) {
1198                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1199
1200                                         } else {
1201                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1202                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1203                                         }
1204                                 }
1205                         }
1206                         break;
1207
1208                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1209                                       /* val.l = constant                             */
1210
1211                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1212                         if (iptr->dst->flags & INMEMORY) {
1213                                 if (src->flags & INMEMORY) {
1214                                         if (x86_64_is_imm32(iptr->val.l)) {
1215                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1216
1217                                         } else {
1218                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1219                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1220                                         }
1221                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1222
1223                                 } else {
1224                                         if (x86_64_is_imm32(iptr->val.l)) {
1225                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1226
1227                                         } else {
1228                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1229                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1230                                         }
1231                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1232                                 }
1233
1234                         } else {
1235                                 if (src->flags & INMEMORY) {
1236                                         if (x86_64_is_imm32(iptr->val.l)) {
1237                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1238
1239                                         } else {
1240                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1241                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1242                                         }
1243
1244                                 } else {
1245                                         /* should match in many cases */
1246                                         if (iptr->val.l == 2) {
1247                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1248                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1249
1250                                         } else {
1251                                                 if (x86_64_is_imm32(iptr->val.l)) {
1252                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1253
1254                                                 } else {
1255                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1256                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1257                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1258                                                 }
1259                                         }
1260                                 }
1261                         }
1262                         break;
1263
1264                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1265
1266                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1267                 if (src->prev->flags & INMEMORY) {
1268                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1269
1270                         } else {
1271                                 M_INTMOVE(src->prev->regoff, RAX);
1272                         }
1273
1274                         if (src->flags & INMEMORY) {
1275                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1276
1277                         } else {
1278                                 M_INTMOVE(src->regoff, REG_ITMP3);
1279                         }
1280                         gen_div_check(src);
1281
1282                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1283                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1284                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1285                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1286
1287                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1288                         x86_64_cltd(cd);
1289                         x86_64_idivl_reg(cd, REG_ITMP3);
1290
1291                         if (iptr->dst->flags & INMEMORY) {
1292                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1293                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1294
1295                         } else {
1296                                 M_INTMOVE(RAX, iptr->dst->regoff);
1297
1298                                 if (iptr->dst->regoff != RDX) {
1299                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1300                                 }
1301                         }
1302                         break;
1303
1304                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1305
1306                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1307                         if (src->prev->flags & INMEMORY) {
1308                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1309
1310                         } else {
1311                                 M_INTMOVE(src->prev->regoff, RAX);
1312                         }
1313
1314                         if (src->flags & INMEMORY) {
1315                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1316
1317                         } else {
1318                                 M_INTMOVE(src->regoff, REG_ITMP3);
1319                         }
1320                         gen_div_check(src);
1321
1322                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1323                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1324                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1325                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1326                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1327
1328                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1329                         x86_64_cltd(cd);
1330                         x86_64_idivl_reg(cd, REG_ITMP3);
1331
1332                         if (iptr->dst->flags & INMEMORY) {
1333                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1334                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1335
1336                         } else {
1337                                 M_INTMOVE(RDX, iptr->dst->regoff);
1338
1339                                 if (iptr->dst->regoff != RDX) {
1340                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1341                                 }
1342                         }
1343                         break;
1344
1345                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1346                                       /* val.i = constant                             */
1347
1348                         var_to_reg_int(s1, src, REG_ITMP1);
1349                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1350                         M_INTMOVE(s1, REG_ITMP1);
1351                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1352                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1353                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1354                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1355                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1356                         store_reg_to_var_int(iptr->dst, d);
1357                         break;
1358
1359                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1360                                       /* val.i = constant                             */
1361
1362                         var_to_reg_int(s1, src, REG_ITMP1);
1363                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1364                         M_INTMOVE(s1, REG_ITMP1);
1365                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1366                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1367                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1368                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1369                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1370                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1371                         store_reg_to_var_int(iptr->dst, d);
1372                         break;
1373
1374
1375                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1376
1377                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1378                 if (src->prev->flags & INMEMORY) {
1379                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1380
1381                         } else {
1382                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1383                         }
1384
1385                         if (src->flags & INMEMORY) {
1386                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1387
1388                         } else {
1389                                 M_INTMOVE(src->regoff, REG_ITMP3);
1390                         }
1391                         gen_div_check(src);
1392
1393                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1394                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1395                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1396                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1397                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1398
1399                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1400                         x86_64_cqto(cd);
1401                         x86_64_idiv_reg(cd, REG_ITMP3);
1402
1403                         if (iptr->dst->flags & INMEMORY) {
1404                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1405                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1406
1407                         } else {
1408                                 M_INTMOVE(RAX, iptr->dst->regoff);
1409
1410                                 if (iptr->dst->regoff != RDX) {
1411                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1412                                 }
1413                         }
1414                         break;
1415
1416                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1417
1418                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1419                         if (src->prev->flags & INMEMORY) {
1420                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1421
1422                         } else {
1423                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1424                         }
1425
1426                         if (src->flags & INMEMORY) {
1427                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1428
1429                         } else {
1430                                 M_INTMOVE(src->regoff, REG_ITMP3);
1431                         }
1432                         gen_div_check(src);
1433
1434                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1435                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1436                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1437                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1438                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1439                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1440
1441                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1442                         x86_64_cqto(cd);
1443                         x86_64_idiv_reg(cd, REG_ITMP3);
1444
1445                         if (iptr->dst->flags & INMEMORY) {
1446                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1447                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1448
1449                         } else {
1450                                 M_INTMOVE(RDX, iptr->dst->regoff);
1451
1452                                 if (iptr->dst->regoff != RDX) {
1453                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1454                                 }
1455                         }
1456                         break;
1457
1458                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1459                                       /* val.i = constant                             */
1460
1461                         var_to_reg_int(s1, src, REG_ITMP1);
1462                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1463                         M_INTMOVE(s1, REG_ITMP1);
1464                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1465                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1466                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1467                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1468                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1469                         store_reg_to_var_int(iptr->dst, d);
1470                         break;
1471
1472                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1473                                       /* val.l = constant                             */
1474
1475                         var_to_reg_int(s1, src, REG_ITMP1);
1476                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1477                         M_INTMOVE(s1, REG_ITMP1);
1478                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1479                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1480                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1481                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1482                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1483                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1484                         store_reg_to_var_int(iptr->dst, d);
1485                         break;
1486
1487                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1488
1489                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1490                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1491                         break;
1492
1493                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1494                                       /* val.i = constant                             */
1495
1496                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1497                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1498                         break;
1499
1500                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1501
1502                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1503                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1504                         break;
1505
1506                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1507                                       /* val.i = constant                             */
1508
1509                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1510                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1511                         break;
1512
1513                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1514
1515                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1516                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1517                         break;
1518
1519                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1520                                       /* val.i = constant                             */
1521
1522                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1523                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1524                         break;
1525
1526                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1527
1528                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1529                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1530                         break;
1531
1532         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1533                                           /* val.i = constant                             */
1534
1535                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1536                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1537                         break;
1538
1539                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1540
1541                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1542                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1543                         break;
1544
1545                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1546                                       /* val.i = constant                             */
1547
1548                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1549                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1550                         break;
1551
1552                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1553
1554                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1555                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1556                         break;
1557
1558                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1559                                       /* val.l = constant                             */
1560
1561                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1562                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1563                         break;
1564
1565                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1566
1567                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1568                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1569                         break;
1570
1571                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1572                                       /* val.i = constant                             */
1573
1574                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1575                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1576                         break;
1577
1578                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1579
1580                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1581                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1582                         break;
1583
1584                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1585                                       /* val.l = constant                             */
1586
1587                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1588                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1589                         break;
1590
1591                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1592
1593                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1594                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1595                         break;
1596
1597                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1598                                       /* val.i = constant                             */
1599
1600                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1601                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1602                         break;
1603
1604                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1605
1606                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1607                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1608                         break;
1609
1610                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1611                                       /* val.l = constant                             */
1612
1613                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1614                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1615                         break;
1616
1617                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1618
1619                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1620                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1621                         break;
1622
1623                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1624                                       /* val.i = constant                             */
1625
1626                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1627                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1628                         break;
1629
1630                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1631
1632                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1633                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1634                         break;
1635
1636                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1637                                       /* val.l = constant                             */
1638
1639                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1640                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1641                         break;
1642
1643
1644                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1645                                       /* op1 = variable, val.i = constant             */
1646
1647                         /* using inc and dec is definitely faster than add -- tested      */
1648                         /* with sieve                                                     */
1649
1650                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1651                         d = var->regoff;
1652                         if (var->flags & INMEMORY) {
1653                                 if (iptr->val.i == 1) {
1654                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1655
1656                                 } else if (iptr->val.i == -1) {
1657                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1658
1659                                 } else {
1660                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1661                                 }
1662
1663                         } else {
1664                                 if (iptr->val.i == 1) {
1665                                         x86_64_incl_reg(cd, d);
1666
1667                                 } else if (iptr->val.i == -1) {
1668                                         x86_64_decl_reg(cd, d);
1669
1670                                 } else {
1671                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1672                                 }
1673                         }
1674                         break;
1675
1676
1677                 /* floating operations ************************************************/
1678
1679                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1680
1681                         var_to_reg_flt(s1, src, REG_FTMP1);
1682                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1683                         a = dseg_adds4(cd, 0x80000000);
1684                         M_FLTMOVE(s1, d);
1685                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1686                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1687                         store_reg_to_var_flt(iptr->dst, d);
1688                         break;
1689
1690                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1691
1692                         var_to_reg_flt(s1, src, REG_FTMP1);
1693                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1694                         a = dseg_adds8(cd, 0x8000000000000000);
1695                         M_FLTMOVE(s1, d);
1696                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1697                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1698                         store_reg_to_var_flt(iptr->dst, d);
1699                         break;
1700
1701                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1702
1703                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1704                         var_to_reg_flt(s2, src, REG_FTMP2);
1705                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1706                         if (s1 == d) {
1707                                 x86_64_addss_reg_reg(cd, s2, d);
1708                         } else if (s2 == d) {
1709                                 x86_64_addss_reg_reg(cd, s1, d);
1710                         } else {
1711                                 M_FLTMOVE(s1, d);
1712                                 x86_64_addss_reg_reg(cd, s2, d);
1713                         }
1714                         store_reg_to_var_flt(iptr->dst, d);
1715                         break;
1716
1717                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1718
1719                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1720                         var_to_reg_flt(s2, src, REG_FTMP2);
1721                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1722                         if (s1 == d) {
1723                                 x86_64_addsd_reg_reg(cd, s2, d);
1724                         } else if (s2 == d) {
1725                                 x86_64_addsd_reg_reg(cd, s1, d);
1726                         } else {
1727                                 M_FLTMOVE(s1, d);
1728                                 x86_64_addsd_reg_reg(cd, s2, d);
1729                         }
1730                         store_reg_to_var_flt(iptr->dst, d);
1731                         break;
1732
1733                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1734
1735                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1736                         var_to_reg_flt(s2, src, REG_FTMP2);
1737                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1738                         if (s2 == d) {
1739                                 M_FLTMOVE(s2, REG_FTMP2);
1740                                 s2 = REG_FTMP2;
1741                         }
1742                         M_FLTMOVE(s1, d);
1743                         x86_64_subss_reg_reg(cd, s2, d);
1744                         store_reg_to_var_flt(iptr->dst, d);
1745                         break;
1746
1747                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1748
1749                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1750                         var_to_reg_flt(s2, src, REG_FTMP2);
1751                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1752                         if (s2 == d) {
1753                                 M_FLTMOVE(s2, REG_FTMP2);
1754                                 s2 = REG_FTMP2;
1755                         }
1756                         M_FLTMOVE(s1, d);
1757                         x86_64_subsd_reg_reg(cd, s2, d);
1758                         store_reg_to_var_flt(iptr->dst, d);
1759                         break;
1760
1761                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1762
1763                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1764                         var_to_reg_flt(s2, src, REG_FTMP2);
1765                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1766                         if (s1 == d) {
1767                                 x86_64_mulss_reg_reg(cd, s2, d);
1768                         } else if (s2 == d) {
1769                                 x86_64_mulss_reg_reg(cd, s1, d);
1770                         } else {
1771                                 M_FLTMOVE(s1, d);
1772                                 x86_64_mulss_reg_reg(cd, s2, d);
1773                         }
1774                         store_reg_to_var_flt(iptr->dst, d);
1775                         break;
1776
1777                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1778
1779                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1780                         var_to_reg_flt(s2, src, REG_FTMP2);
1781                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1782                         if (s1 == d) {
1783                                 x86_64_mulsd_reg_reg(cd, s2, d);
1784                         } else if (s2 == d) {
1785                                 x86_64_mulsd_reg_reg(cd, s1, d);
1786                         } else {
1787                                 M_FLTMOVE(s1, d);
1788                                 x86_64_mulsd_reg_reg(cd, s2, d);
1789                         }
1790                         store_reg_to_var_flt(iptr->dst, d);
1791                         break;
1792
1793                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1794
1795                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1796                         var_to_reg_flt(s2, src, REG_FTMP2);
1797                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1798                         if (s2 == d) {
1799                                 M_FLTMOVE(s2, REG_FTMP2);
1800                                 s2 = REG_FTMP2;
1801                         }
1802                         M_FLTMOVE(s1, d);
1803                         x86_64_divss_reg_reg(cd, s2, d);
1804                         store_reg_to_var_flt(iptr->dst, d);
1805                         break;
1806
1807                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1808
1809                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1810                         var_to_reg_flt(s2, src, REG_FTMP2);
1811                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1812                         if (s2 == d) {
1813                                 M_FLTMOVE(s2, REG_FTMP2);
1814                                 s2 = REG_FTMP2;
1815                         }
1816                         M_FLTMOVE(s1, d);
1817                         x86_64_divsd_reg_reg(cd, s2, d);
1818                         store_reg_to_var_flt(iptr->dst, d);
1819                         break;
1820
1821                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1822
1823                         var_to_reg_int(s1, src, REG_ITMP1);
1824                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1825                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1826                         store_reg_to_var_flt(iptr->dst, d);
1827                         break;
1828
1829                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1830
1831                         var_to_reg_int(s1, src, REG_ITMP1);
1832                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1833                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1834                         store_reg_to_var_flt(iptr->dst, d);
1835                         break;
1836
1837                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1838
1839                         var_to_reg_int(s1, src, REG_ITMP1);
1840                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1841                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1842                         store_reg_to_var_flt(iptr->dst, d);
1843                         break;
1844
1845                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1846
1847                         var_to_reg_int(s1, src, REG_ITMP1);
1848                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1849                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1850                         store_reg_to_var_flt(iptr->dst, d);
1851                         break;
1852
1853                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1854
1855                         var_to_reg_flt(s1, src, REG_FTMP1);
1856                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1857                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1858                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1859                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1860                         x86_64_jcc(cd, X86_64_CC_NE, a);
1861                         M_FLTMOVE(s1, REG_FTMP1);
1862                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1863                         x86_64_call_reg(cd, REG_ITMP2);
1864                         M_INTMOVE(REG_RESULT, d);
1865                         store_reg_to_var_int(iptr->dst, d);
1866                         break;
1867
1868                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1869
1870                         var_to_reg_flt(s1, src, REG_FTMP1);
1871                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1872                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1873                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1874                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1875                         x86_64_jcc(cd, X86_64_CC_NE, a);
1876                         M_FLTMOVE(s1, REG_FTMP1);
1877                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1878                         x86_64_call_reg(cd, REG_ITMP2);
1879                         M_INTMOVE(REG_RESULT, d);
1880                         store_reg_to_var_int(iptr->dst, d);
1881                         break;
1882
1883                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1884
1885                         var_to_reg_flt(s1, src, REG_FTMP1);
1886                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1887                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1888                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1889                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1890                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1891                         x86_64_jcc(cd, X86_64_CC_NE, a);
1892                         M_FLTMOVE(s1, REG_FTMP1);
1893                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1894                         x86_64_call_reg(cd, REG_ITMP2);
1895                         M_INTMOVE(REG_RESULT, d);
1896                         store_reg_to_var_int(iptr->dst, d);
1897                         break;
1898
1899                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1900
1901                         var_to_reg_flt(s1, src, REG_FTMP1);
1902                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1903                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1904                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1905                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1906                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1907                         x86_64_jcc(cd, X86_64_CC_NE, a);
1908                         M_FLTMOVE(s1, REG_FTMP1);
1909                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1910                         x86_64_call_reg(cd, REG_ITMP2);
1911                         M_INTMOVE(REG_RESULT, d);
1912                         store_reg_to_var_int(iptr->dst, d);
1913                         break;
1914
1915                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1916
1917                         var_to_reg_flt(s1, src, REG_FTMP1);
1918                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1919                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1920                         store_reg_to_var_flt(iptr->dst, d);
1921                         break;
1922
1923                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1924
1925                         var_to_reg_flt(s1, src, REG_FTMP1);
1926                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1927                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1928                         store_reg_to_var_flt(iptr->dst, d);
1929                         break;
1930
1931                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1932                                           /* == => 0, < => 1, > => -1 */
1933
1934                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1935                         var_to_reg_flt(s2, src, REG_FTMP2);
1936                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1937                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1938                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1939                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1940                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1941                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1942                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1943                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1944                         store_reg_to_var_int(iptr->dst, d);
1945                         break;
1946
1947                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1948                                           /* == => 0, < => 1, > => -1 */
1949
1950                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1951                         var_to_reg_flt(s2, src, REG_FTMP2);
1952                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1953                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1954                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1955                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1956                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1957                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1958                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1959                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1960                         store_reg_to_var_int(iptr->dst, d);
1961                         break;
1962
1963                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1964                                           /* == => 0, < => 1, > => -1 */
1965
1966                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1967                         var_to_reg_flt(s2, src, REG_FTMP2);
1968                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1969                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1970                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1971                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1972                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1973                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1974                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1975                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1976                         store_reg_to_var_int(iptr->dst, d);
1977                         break;
1978
1979                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1980                                           /* == => 0, < => 1, > => -1 */
1981
1982                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1983                         var_to_reg_flt(s2, src, REG_FTMP2);
1984                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1985                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1986                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1987                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1988                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1989                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1990                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1991                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1992                         store_reg_to_var_int(iptr->dst, d);
1993                         break;
1994
1995
1996                 /* memory operations **************************************************/
1997
1998                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1999
2000                         var_to_reg_int(s1, src, REG_ITMP1);
2001                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2002                         gen_nullptr_check(s1);
2003                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
2004                         store_reg_to_var_int(iptr->dst, d);
2005                         break;
2006
2007                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2008
2009                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2010                         var_to_reg_int(s2, src, REG_ITMP2);
2011                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2012                         if (iptr->op1 == 0) {
2013                                 gen_nullptr_check(s1);
2014                                 gen_bound_check;
2015                         }
2016                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2017                         store_reg_to_var_int(iptr->dst, d);
2018                         break;
2019
2020                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2021
2022                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2023                         var_to_reg_int(s2, src, REG_ITMP2);
2024                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2025                         if (iptr->op1 == 0) {
2026                                 gen_nullptr_check(s1);
2027                                 gen_bound_check;
2028                         }
2029                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2030                         store_reg_to_var_int(iptr->dst, d);
2031                         break;
2032
2033                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2034
2035                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2036                         var_to_reg_int(s2, src, REG_ITMP2);
2037                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2038                         if (iptr->op1 == 0) {
2039                                 gen_nullptr_check(s1);
2040                                 gen_bound_check;
2041                         }
2042                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2043                         store_reg_to_var_int(iptr->dst, d);
2044                         break;
2045
2046                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2047
2048                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2049                         var_to_reg_int(s2, src, REG_ITMP2);
2050                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2051                         if (iptr->op1 == 0) {
2052                                 gen_nullptr_check(s1);
2053                                 gen_bound_check;
2054                         }
2055                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2056                         store_reg_to_var_flt(iptr->dst, d);
2057                         break;
2058
2059                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2060
2061                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2062                         var_to_reg_int(s2, src, REG_ITMP2);
2063                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2064                         if (iptr->op1 == 0) {
2065                                 gen_nullptr_check(s1);
2066                                 gen_bound_check;
2067                         }
2068                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2069                         store_reg_to_var_flt(iptr->dst, d);
2070                         break;
2071
2072                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2073
2074                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2075                         var_to_reg_int(s2, src, REG_ITMP2);
2076                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2077                         if (iptr->op1 == 0) {
2078                                 gen_nullptr_check(s1);
2079                                 gen_bound_check;
2080                         }
2081                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2082                         store_reg_to_var_int(iptr->dst, d);
2083                         break;
2084
2085                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2086
2087                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2088                         var_to_reg_int(s2, src, REG_ITMP2);
2089                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2090                         if (iptr->op1 == 0) {
2091                                 gen_nullptr_check(s1);
2092                                 gen_bound_check;
2093                         }
2094                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2095                         store_reg_to_var_int(iptr->dst, d);
2096                         break;
2097
2098                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2099
2100                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2101                         var_to_reg_int(s2, src, REG_ITMP2);
2102                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2103                         if (iptr->op1 == 0) {
2104                                 gen_nullptr_check(s1);
2105                                 gen_bound_check;
2106                         }
2107                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2108                         store_reg_to_var_int(iptr->dst, d);
2109                         break;
2110
2111
2112                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2113
2114                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2115                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2116                         if (iptr->op1 == 0) {
2117                                 gen_nullptr_check(s1);
2118                                 gen_bound_check;
2119                         }
2120                         var_to_reg_int(s3, src, REG_ITMP3);
2121                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2122                         break;
2123
2124                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2125
2126                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2127                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2128                         if (iptr->op1 == 0) {
2129                                 gen_nullptr_check(s1);
2130                                 gen_bound_check;
2131                         }
2132                         var_to_reg_int(s3, src, REG_ITMP3);
2133                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2134                         break;
2135
2136                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2137
2138                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2139                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2140                         if (iptr->op1 == 0) {
2141                                 gen_nullptr_check(s1);
2142                                 gen_bound_check;
2143                         }
2144                         var_to_reg_int(s3, src, REG_ITMP3);
2145                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2146                         break;
2147
2148                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2149
2150                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2151                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2152                         if (iptr->op1 == 0) {
2153                                 gen_nullptr_check(s1);
2154                                 gen_bound_check;
2155                         }
2156                         var_to_reg_flt(s3, src, REG_FTMP3);
2157                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2158                         break;
2159
2160                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2161
2162                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2163                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2164                         if (iptr->op1 == 0) {
2165                                 gen_nullptr_check(s1);
2166                                 gen_bound_check;
2167                         }
2168                         var_to_reg_flt(s3, src, REG_FTMP3);
2169                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2170                         break;
2171
2172                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2173
2174                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2175                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2176                         if (iptr->op1 == 0) {
2177                                 gen_nullptr_check(s1);
2178                                 gen_bound_check;
2179                         }
2180                         var_to_reg_int(s3, src, REG_ITMP3);
2181                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2182                         break;
2183
2184                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2185
2186                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2187                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2188                         if (iptr->op1 == 0) {
2189                                 gen_nullptr_check(s1);
2190                                 gen_bound_check;
2191                         }
2192                         var_to_reg_int(s3, src, REG_ITMP3);
2193                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2194                         break;
2195
2196                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2197
2198                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2199                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2200                         if (iptr->op1 == 0) {
2201                                 gen_nullptr_check(s1);
2202                                 gen_bound_check;
2203                         }
2204                         var_to_reg_int(s3, src, REG_ITMP3);
2205                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2206                         break;
2207
2208                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2209
2210                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2211                         var_to_reg_int(s2, src, REG_ITMP2);
2212                         if (iptr->op1 == 0) {
2213                                 gen_nullptr_check(s1);
2214                                 gen_bound_check;
2215                         }
2216                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2217                         break;
2218
2219                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2220
2221                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2222                         var_to_reg_int(s2, src, REG_ITMP2);
2223                         if (iptr->op1 == 0) {
2224                                 gen_nullptr_check(s1);
2225                                 gen_bound_check;
2226                         }
2227
2228                         if (x86_64_is_imm32(iptr->val.l)) {
2229                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2230
2231                         } else {
2232                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2233                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2234                         }
2235                         break;
2236
2237                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2238
2239                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2240                         var_to_reg_int(s2, src, REG_ITMP2);
2241                         if (iptr->op1 == 0) {
2242                                 gen_nullptr_check(s1);
2243                                 gen_bound_check;
2244                         }
2245                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2246                         break;
2247
2248                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2249
2250                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2251                         var_to_reg_int(s2, src, REG_ITMP2);
2252                         if (iptr->op1 == 0) {
2253                                 gen_nullptr_check(s1);
2254                                 gen_bound_check;
2255                         }
2256                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2257                         break;
2258
2259                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2260
2261                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2262                         var_to_reg_int(s2, src, REG_ITMP2);
2263                         if (iptr->op1 == 0) {
2264                                 gen_nullptr_check(s1);
2265                                 gen_bound_check;
2266                         }
2267                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2268                         break;
2269
2270                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2271
2272                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2273                         var_to_reg_int(s2, src, REG_ITMP2);
2274                         if (iptr->op1 == 0) {
2275                                 gen_nullptr_check(s1);
2276                                 gen_bound_check;
2277                         }
2278                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2279                         break;
2280
2281
2282                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2283                                       /* op1 = type, val.a = field address            */
2284
2285                         /* If the static fields' class is not yet initialized, we do it   */
2286                         /* now. The call code is generated later.                         */
2287                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2288                                 codegen_addclinitref(cd, cd->mcodeptr, ((fieldinfo *) iptr->val.a)->class);
2289
2290                                 /* This is just for debugging purposes. Is very difficult to  */
2291                                 /* read patched code. Here we patch the following 5 nop's     */
2292                                 /* so that the real code keeps untouched.                     */
2293                                 if (showdisassemble) {
2294                                         x86_64_nop(cd);
2295                                         x86_64_nop(cd);
2296                                         x86_64_nop(cd);
2297                                         x86_64_nop(cd);
2298                                         x86_64_nop(cd);
2299                                 }
2300                         }
2301
2302                         /* This approach is much faster than moving the field address     */
2303                         /* inline into a register. */
2304                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2305                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2306                         switch (iptr->op1) {
2307                         case TYPE_INT:
2308                                 var_to_reg_int(s2, src, REG_ITMP1);
2309                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2310                                 break;
2311                         case TYPE_LNG:
2312                         case TYPE_ADR:
2313                                 var_to_reg_int(s2, src, REG_ITMP1);
2314                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2315                                 break;
2316                         case TYPE_FLT:
2317                                 var_to_reg_flt(s2, src, REG_FTMP1);
2318                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2319                                 break;
2320                         case TYPE_DBL:
2321                                 var_to_reg_flt(s2, src, REG_FTMP1);
2322                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2323                                 break;
2324                         default:
2325                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2326                                                                                    "Unknown PUTSTATIC operand type %d",
2327                                                                                    iptr->op1);
2328                         }
2329                         break;
2330
2331                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2332                                       /* op1 = type, val.a = field address            */
2333
2334                         /* If the static fields' class is not yet initialized, we do it   */
2335                         /* now. The call code is generated later.                         */
2336                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2337                                 codegen_addclinitref(cd, cd->mcodeptr, ((fieldinfo *) iptr->val.a)->class);
2338
2339                                 /* This is just for debugging purposes. Is very difficult to  */
2340                                 /* read patched code. Here we patch the following 5 nop's     */
2341                                 /* so that the real code keeps untouched.                     */
2342                                 if (showdisassemble) {
2343                                         x86_64_nop(cd);
2344                                         x86_64_nop(cd);
2345                                         x86_64_nop(cd);
2346                                         x86_64_nop(cd);
2347                                         x86_64_nop(cd);
2348                                 }
2349                         }
2350
2351                         /* This approach is much faster than moving the field address     */
2352                         /* inline into a register. */
2353                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2354                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2355                         switch (iptr->op1) {
2356                         case TYPE_INT:
2357                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2358                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2359                                 store_reg_to_var_int(iptr->dst, d);
2360                                 break;
2361                         case TYPE_LNG:
2362                         case TYPE_ADR:
2363                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2364                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2365                                 store_reg_to_var_int(iptr->dst, d);
2366                                 break;
2367                         case TYPE_FLT:
2368                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2369                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2370                                 store_reg_to_var_flt(iptr->dst, d);
2371                                 break;
2372                         case TYPE_DBL:
2373                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2374                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2375                                 store_reg_to_var_flt(iptr->dst, d);
2376                                 break;
2377                         default:
2378                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2379                                                                                    "Unknown GETSTATIC operand type %d",
2380                                                                                    iptr->op1);
2381                         }
2382                         break;
2383
2384                 case ICMD_PUTFIELD:   /* ..., value  ==> ...                          */
2385                                       /* op1 = type, val.i = field offset             */
2386
2387                         a = ((fieldinfo *)(iptr->val.a))->offset;
2388                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2389                         switch (iptr->op1) {
2390                         case TYPE_INT:
2391                                 var_to_reg_int(s2, src, REG_ITMP2);
2392                                 gen_nullptr_check(s1);
2393                                 x86_64_movl_reg_membase(cd, s2, s1, a);
2394                                 break;
2395                         case TYPE_LNG:
2396                         case TYPE_ADR:
2397                                 var_to_reg_int(s2, src, REG_ITMP2);
2398                                 gen_nullptr_check(s1);
2399                                 x86_64_mov_reg_membase(cd, s2, s1, a);
2400                                 break;
2401                         case TYPE_FLT:
2402                                 var_to_reg_flt(s2, src, REG_FTMP2);
2403                                 gen_nullptr_check(s1);
2404                                 x86_64_movss_reg_membase(cd, s2, s1, a);
2405                                 break;
2406                         case TYPE_DBL:
2407                                 var_to_reg_flt(s2, src, REG_FTMP2);
2408                                 gen_nullptr_check(s1);
2409                                 x86_64_movsd_reg_membase(cd, s2, s1, a);
2410                                 break;
2411                         default:
2412                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2413                                                                                    "Unknown PUTFIELD operand type %d",
2414                                                                                    iptr->op1);
2415                         }
2416                         break;
2417
2418                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2419                                       /* op1 = type, val.i = field offset             */
2420
2421                         a = ((fieldinfo *)(iptr->val.a))->offset;
2422                         var_to_reg_int(s1, src, REG_ITMP1);
2423                         switch (iptr->op1) {
2424                         case TYPE_INT:
2425                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2426                                 gen_nullptr_check(s1);
2427                                 x86_64_movl_membase_reg(cd, s1, a, d);
2428                                 store_reg_to_var_int(iptr->dst, d);
2429                                 break;
2430                         case TYPE_LNG:
2431                         case TYPE_ADR:
2432                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2433                                 gen_nullptr_check(s1);
2434                                 x86_64_mov_membase_reg(cd, s1, a, d);
2435                                 store_reg_to_var_int(iptr->dst, d);
2436                                 break;
2437                         case TYPE_FLT:
2438                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2439                                 gen_nullptr_check(s1);
2440                                 x86_64_movss_membase_reg(cd, s1, a, d);
2441                                 store_reg_to_var_flt(iptr->dst, d);
2442                                 break;
2443                         case TYPE_DBL:
2444                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2445                                 gen_nullptr_check(s1);
2446                                 x86_64_movsd_membase_reg(cd, s1, a, d);
2447                                 store_reg_to_var_flt(iptr->dst, d);
2448                                 break;
2449                         default:
2450                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2451                                                                                    "Unknown GETFIELD operand type %d",
2452                                                                                    iptr->op1);
2453                         }
2454                         break;
2455
2456
2457                 /* branch operations **************************************************/
2458
2459                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2460
2461                         var_to_reg_int(s1, src, REG_ITMP1);
2462                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2463
2464                         x86_64_call_imm(cd, 0); /* passing exception pointer              */
2465                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2466
2467                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
2468                         x86_64_jmp_reg(cd, REG_ITMP3);
2469                         ALIGNCODENOP;
2470                         break;
2471
2472                 case ICMD_GOTO:         /* ... ==> ...                                */
2473                                         /* op1 = target JavaVM pc                     */
2474
2475                         x86_64_jmp_imm(cd, 0);
2476                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2477                         ALIGNCODENOP;
2478                         break;
2479
2480                 case ICMD_JSR:          /* ... ==> ...                                */
2481                                         /* op1 = target JavaVM pc                     */
2482
2483                         x86_64_call_imm(cd, 0);
2484                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2485                         break;
2486
2487                 case ICMD_RET:          /* ... ==> ...                                */
2488                                         /* op1 = local variable                       */
2489
2490                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2491                         var_to_reg_int(s1, var, REG_ITMP1);
2492                         x86_64_jmp_reg(cd, s1);
2493                         break;
2494
2495                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2496                                         /* op1 = target JavaVM pc                     */
2497
2498                         if (src->flags & INMEMORY) {
2499                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2500
2501                         } else {
2502                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2503                         }
2504                         x86_64_jcc(cd, X86_64_CC_E, 0);
2505                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2506                         break;
2507
2508                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2509                                         /* op1 = target JavaVM pc                     */
2510
2511                         if (src->flags & INMEMORY) {
2512                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2513
2514                         } else {
2515                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2516                         }
2517                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2518                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2519                         break;
2520
2521                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2522                                         /* op1 = target JavaVM pc, val.i = constant   */
2523
2524                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2525                         break;
2526
2527                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2528                                         /* op1 = target JavaVM pc, val.i = constant   */
2529
2530                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2531                         break;
2532
2533                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2534                                         /* op1 = target JavaVM pc, val.i = constant   */
2535
2536                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2537                         break;
2538
2539                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2540                                         /* op1 = target JavaVM pc, val.i = constant   */
2541
2542                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2543                         break;
2544
2545                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2546                                         /* op1 = target JavaVM pc, val.i = constant   */
2547
2548                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2549                         break;
2550
2551                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2552                                         /* op1 = target JavaVM pc, val.i = constant   */
2553
2554                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2555                         break;
2556
2557                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2558                                         /* op1 = target JavaVM pc, val.l = constant   */
2559
2560                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2561                         break;
2562
2563                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2564                                         /* op1 = target JavaVM pc, val.l = constant   */
2565
2566                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2567                         break;
2568
2569                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2570                                         /* op1 = target JavaVM pc, val.l = constant   */
2571
2572                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2573                         break;
2574
2575                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2576                                         /* op1 = target JavaVM pc, val.l = constant   */
2577
2578                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2579                         break;
2580
2581                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2582                                         /* op1 = target JavaVM pc, val.l = constant   */
2583
2584                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2585                         break;
2586
2587                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2588                                         /* op1 = target JavaVM pc, val.l = constant   */
2589
2590                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2591                         break;
2592
2593                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2594                                         /* op1 = target JavaVM pc                     */
2595
2596                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2597                         break;
2598
2599                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2600                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2601
2602                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2603                         break;
2604
2605                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2606                                         /* op1 = target JavaVM pc                     */
2607
2608                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2609                         break;
2610
2611                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2612                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2613
2614                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2615                         break;
2616
2617                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2618                                         /* op1 = target JavaVM pc                     */
2619
2620                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2621                         break;
2622
2623                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2624                                     /* op1 = target JavaVM pc                     */
2625
2626                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2627                         break;
2628
2629                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2630                                         /* op1 = target JavaVM pc                     */
2631
2632                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2633                         break;
2634
2635                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2636                                 /* op1 = target JavaVM pc                     */
2637
2638                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2639                         break;
2640
2641                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2642                                         /* op1 = target JavaVM pc                     */
2643
2644                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2645                         break;
2646
2647                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2648                                         /* op1 = target JavaVM pc                     */
2649
2650                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2651                         break;
2652
2653                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2654                                         /* op1 = target JavaVM pc                     */
2655
2656                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2657                         break;
2658
2659                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2660                                     /* op1 = target JavaVM pc                     */
2661
2662                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2663                         break;
2664
2665                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2666
2667                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2668                         break;
2669
2670                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2671                                         /* val.i = constant                           */
2672
2673                         var_to_reg_int(s1, src, REG_ITMP1);
2674                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2675                         s3 = iptr->val.i;
2676                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2677                                 if (s1 == d) {
2678                                         M_INTMOVE(s1, REG_ITMP1);
2679                                         s1 = REG_ITMP1;
2680                                 }
2681                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2682                         }
2683                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2684                         x86_64_testl_reg_reg(cd, s1, s1);
2685                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2686                         store_reg_to_var_int(iptr->dst, d);
2687                         break;
2688
2689                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2690                                         /* val.i = constant                           */
2691
2692                         var_to_reg_int(s1, src, REG_ITMP1);
2693                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2694                         s3 = iptr->val.i;
2695                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2696                                 if (s1 == d) {
2697                                         M_INTMOVE(s1, REG_ITMP1);
2698                                         s1 = REG_ITMP1;
2699                                 }
2700                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2701                         }
2702                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2703                         x86_64_testl_reg_reg(cd, s1, s1);
2704                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2705                         store_reg_to_var_int(iptr->dst, d);
2706                         break;
2707
2708                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2709                                         /* val.i = constant                           */
2710
2711                         var_to_reg_int(s1, src, REG_ITMP1);
2712                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2713                         s3 = iptr->val.i;
2714                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2715                                 if (s1 == d) {
2716                                         M_INTMOVE(s1, REG_ITMP1);
2717                                         s1 = REG_ITMP1;
2718                                 }
2719                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2720                         }
2721                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2722                         x86_64_testl_reg_reg(cd, s1, s1);
2723                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2724                         store_reg_to_var_int(iptr->dst, d);
2725                         break;
2726
2727                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2728                                         /* val.i = constant                           */
2729
2730                         var_to_reg_int(s1, src, REG_ITMP1);
2731                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2732                         s3 = iptr->val.i;
2733                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2734                                 if (s1 == d) {
2735                                         M_INTMOVE(s1, REG_ITMP1);
2736                                         s1 = REG_ITMP1;
2737                                 }
2738                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2739                         }
2740                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2741                         x86_64_testl_reg_reg(cd, s1, s1);
2742                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2743                         store_reg_to_var_int(iptr->dst, d);
2744                         break;
2745
2746                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2747                                         /* val.i = constant                           */
2748
2749                         var_to_reg_int(s1, src, REG_ITMP1);
2750                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2751                         s3 = iptr->val.i;
2752                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2753                                 if (s1 == d) {
2754                                         M_INTMOVE(s1, REG_ITMP1);
2755                                         s1 = REG_ITMP1;
2756                                 }
2757                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2758                         }
2759                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2760                         x86_64_testl_reg_reg(cd, s1, s1);
2761                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2762                         store_reg_to_var_int(iptr->dst, d);
2763                         break;
2764
2765                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2766                                         /* val.i = constant                           */
2767
2768                         var_to_reg_int(s1, src, REG_ITMP1);
2769                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2770                         s3 = iptr->val.i;
2771                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2772                                 if (s1 == d) {
2773                                         M_INTMOVE(s1, REG_ITMP1);
2774                                         s1 = REG_ITMP1;
2775                                 }
2776                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2777                         }
2778                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2779                         x86_64_testl_reg_reg(cd, s1, s1);
2780                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2781                         store_reg_to_var_int(iptr->dst, d);
2782                         break;
2783
2784
2785                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2786                 case ICMD_LRETURN:
2787                 case ICMD_ARETURN:
2788
2789                         var_to_reg_int(s1, src, REG_RESULT);
2790                         M_INTMOVE(s1, REG_RESULT);
2791
2792 #if defined(USE_THREADS)
2793                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2794                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2795                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->maxmemuse * 8);
2796                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2797                                 x86_64_call_reg(cd, REG_ITMP1);
2798                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_RESULT);
2799                         }
2800 #endif
2801
2802                         goto nowperformreturn;
2803
2804                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2805                 case ICMD_DRETURN:
2806
2807                         var_to_reg_flt(s1, src, REG_FRESULT);
2808                         M_FLTMOVE(s1, REG_FRESULT);
2809
2810 #if defined(USE_THREADS)
2811                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2812                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2813                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->maxmemuse * 8);
2814                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2815                                 x86_64_call_reg(cd, REG_ITMP1);
2816                                 x86_64_movq_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_FRESULT);
2817                         }
2818 #endif
2819
2820                         goto nowperformreturn;
2821
2822                 case ICMD_RETURN:      /* ...  ==> ...                                */
2823
2824 #if defined(USE_THREADS)
2825                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2826                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2827                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2828                                 x86_64_call_reg(cd, REG_ITMP1);
2829                         }
2830 #endif
2831
2832 nowperformreturn:
2833                         {
2834                         s4 i, p;
2835
2836                         p = parentargs_base;
2837
2838                         /* call trace function */
2839                         if (runverbose) {
2840                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2841
2842                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2843                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2844
2845                                 x86_64_mov_imm_reg(cd, (s8) m, rd->argintregs[0]);
2846                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2847                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2848                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2849
2850                                 x86_64_mov_imm_reg(cd, (s8) builtin_displaymethodstop, REG_ITMP1);
2851                                 x86_64_call_reg(cd, REG_ITMP1);
2852
2853                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2854                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2855
2856                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2857                         }
2858
2859                         /* restore saved registers                                        */
2860                         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
2861                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
2862                         }
2863                         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
2864                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
2865                         }
2866
2867                         /* deallocate stack                                               */
2868                         if (parentargs_base) {
2869                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2870                         }
2871
2872                         x86_64_ret(cd);
2873                         ALIGNCODENOP;
2874                         }
2875                         break;
2876
2877
2878                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2879                         {
2880                                 s4 i, l, *s4ptr;
2881                                 void **tptr;
2882
2883                                 tptr = (void **) iptr->target;
2884
2885                                 s4ptr = iptr->val.a;
2886                                 l = s4ptr[1];                          /* low     */
2887                                 i = s4ptr[2];                          /* high    */
2888
2889                                 var_to_reg_int(s1, src, REG_ITMP1);
2890                                 M_INTMOVE(s1, REG_ITMP1);
2891                                 if (l != 0) {
2892                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2893                                 }
2894                                 i = i - l + 1;
2895
2896                 /* range check */
2897                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2898                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2899
2900                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2901                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2902
2903                                 /* build jump table top down and use address of lowest entry */
2904
2905                 /* s4ptr += 3 + i; */
2906                                 tptr += i;
2907
2908                                 while (--i >= 0) {
2909                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
2910                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2911                                         --tptr;
2912                                 }
2913
2914                                 /* length of dataseg after last dseg_addtarget is used by load */
2915
2916                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2917                                 dseg_adddata(cd, cd->mcodeptr);
2918                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2919                                 x86_64_jmp_reg(cd, REG_ITMP1);
2920                                 ALIGNCODENOP;
2921                         }
2922                         break;
2923
2924
2925                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2926                         {
2927                                 s4 i, l, val, *s4ptr;
2928                                 void **tptr;
2929
2930                                 tptr = (void **) iptr->target;
2931
2932                                 s4ptr = iptr->val.a;
2933                                 l = s4ptr[0];                          /* default  */
2934                                 i = s4ptr[1];                          /* count    */
2935
2936                                 MCODECHECK((i<<2)+8);
2937                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
2938                                 while (--i >= 0) {
2939                                         s4ptr += 2;
2940                                         ++tptr;
2941
2942                                         val = s4ptr[0];
2943                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
2944                                         x86_64_jcc(cd, X86_64_CC_E, 0);
2945                                         /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
2946                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2947                                 }
2948
2949                                 x86_64_jmp_imm(cd, 0);
2950                                 /* codegen_addreference(cd, BlockPtrOfPC(l), cd->mcodeptr); */
2951
2952                                 tptr = (void **) iptr->target;
2953                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2954
2955                                 ALIGNCODENOP;
2956                         }
2957                         break;
2958
2959
2960                 case ICMD_BUILTIN3:     /* ..., arg1, arg2, arg3 ==> ...              */
2961                                         /* op1 = return type, val.a = function pointer*/
2962                         s3 = 3;
2963                         goto gen_method;
2964
2965                 case ICMD_BUILTIN2:     /* ..., arg1, arg2 ==> ...                    */
2966                                         /* op1 = return type, val.a = function pointer*/
2967                         s3 = 2;
2968                         goto gen_method;
2969
2970                 case ICMD_BUILTIN1:     /* ..., arg1 ==> ...                          */
2971                                         /* op1 = return type, val.a = function pointer*/
2972                         s3 = 1;
2973                         goto gen_method;
2974
2975                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2976                                         /* op1 = arg count, val.a = method pointer    */
2977
2978                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2979                                         /* op1 = arg count, val.a = method pointer    */
2980
2981                 case ICMD_INVOKEVIRTUAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2982                                         /* op1 = arg count, val.a = method pointer    */
2983
2984                 case ICMD_INVOKEINTERFACE:/*.., objectref, [arg1, [arg2 ...]] ==> ... */
2985                                         /* op1 = arg count, val.a = method pointer    */
2986
2987                         s3 = iptr->op1;
2988
2989 gen_method: {
2990                         methodinfo   *lm;
2991                         classinfo    *ci;
2992                         stackptr      tmpsrc;
2993                         s4 iarg, farg;
2994
2995                         MCODECHECK((s3 << 1) + 64);
2996
2997                         tmpsrc = src;
2998                         s2 = s3;
2999                         iarg = 0;
3000                         farg = 0;
3001
3002                         /* copy arguments to registers or stack location                  */
3003                         for (; --s3 >= 0; src = src->prev) {
3004                                 IS_INT_LNG_TYPE(src->type) ? iarg++ : farg++;
3005                         }
3006
3007                         src = tmpsrc;
3008                         s3 = s2;
3009
3010                         s2 = (iarg > INT_ARG_CNT) ? iarg - INT_ARG_CNT : 0 + (farg > FLT_ARG_CNT) ? farg - FLT_ARG_CNT : 0;
3011
3012                         for (; --s3 >= 0; src = src->prev) {
3013                                 IS_INT_LNG_TYPE(src->type) ? iarg-- : farg--;
3014                                 if (src->varkind == ARGVAR) {
3015                                         if (IS_INT_LNG_TYPE(src->type)) {
3016                                                 if (iarg >= INT_ARG_CNT) {
3017                                                         s2--;
3018                                                 }
3019                                         } else {
3020                                                 if (farg >= FLT_ARG_CNT) {
3021                                                         s2--;
3022                                                 }
3023                                         }
3024                                         continue;
3025                                 }
3026
3027                                 if (IS_INT_LNG_TYPE(src->type)) {
3028                                         if (iarg < INT_ARG_CNT) {
3029                                                 s1 = rd->argintregs[iarg];
3030                                                 var_to_reg_int(d, src, s1);
3031                                                 M_INTMOVE(d, s1);
3032
3033                                         } else {
3034                                                 var_to_reg_int(d, src, REG_ITMP1);
3035                                                 s2--;
3036                                                 x86_64_mov_reg_membase(cd, d, REG_SP, s2 * 8);
3037                                         }
3038
3039                                 } else {
3040                                         if (farg < FLT_ARG_CNT) {
3041                                                 s1 = rd->argfltregs[farg];
3042                                                 var_to_reg_flt(d, src, s1);
3043                                                 M_FLTMOVE(d, s1);
3044
3045                                         } else {
3046                                                 var_to_reg_flt(d, src, REG_FTMP1);
3047                                                 s2--;
3048                                                 x86_64_movq_reg_membase(cd, d, REG_SP, s2 * 8);
3049                                         }
3050                                 }
3051                         } /* end of for */
3052
3053                         lm = iptr->val.a;
3054                         switch (iptr->opc) {
3055                                 case ICMD_BUILTIN3:
3056                                 case ICMD_BUILTIN2:
3057                                 case ICMD_BUILTIN1:
3058
3059                                         a = (s8) lm;
3060                                         d = iptr->op1;
3061
3062                                         x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3063                                         x86_64_call_reg(cd, REG_ITMP1);
3064                                         break;
3065
3066                                 case ICMD_INVOKESTATIC:
3067
3068                                         a = (s8) lm->stubroutine;
3069                                         d = lm->returntype;
3070
3071                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3072                                         x86_64_call_reg(cd, REG_ITMP2);
3073                                         break;
3074
3075                                 case ICMD_INVOKESPECIAL:
3076
3077                                         a = (s8) lm->stubroutine;
3078                                         d = lm->returntype;
3079
3080                                         gen_nullptr_check(rd->argintregs[0]);    /* first argument contains pointer */
3081                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); /* access memory for hardware nullptr */
3082                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3083                                         x86_64_call_reg(cd, REG_ITMP2);
3084                                         break;
3085
3086                                 case ICMD_INVOKEVIRTUAL:
3087
3088                                         d = lm->returntype;
3089
3090                                         gen_nullptr_check(rd->argintregs[0]);
3091                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3092                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, OFFSET(vftbl_t, table[0]) + sizeof(methodptr) * lm->vftblindex, REG_ITMP1);
3093                                         x86_64_call_reg(cd, REG_ITMP1);
3094                                         break;
3095
3096                                 case ICMD_INVOKEINTERFACE:
3097
3098                                         ci = lm->class;
3099                                         d = lm->returntype;
3100
3101                                         gen_nullptr_check(rd->argintregs[0]);
3102                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3103                                         x86_64_mov_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, interfacetable[0]) - sizeof(methodptr) * ci->index, REG_ITMP2);
3104                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, sizeof(methodptr) * (lm - ci->methods), REG_ITMP1);
3105                                         x86_64_call_reg(cd, REG_ITMP1);
3106                                         break;
3107
3108                                 default:
3109                                         d = 0;
3110                                         error("Unkown ICMD-Command: %d", iptr->opc);
3111                                 }
3112
3113                         /* d contains return type */
3114
3115                         if (d != TYPE_VOID) {
3116                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3117                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3118                                         M_INTMOVE(REG_RESULT, s1);
3119                                         store_reg_to_var_int(iptr->dst, s1);
3120
3121                                 } else {
3122                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3123                                         M_FLTMOVE(REG_FRESULT, s1);
3124                                         store_reg_to_var_flt(iptr->dst, s1);
3125                                 }
3126                         }
3127                         }
3128                         break;
3129
3130
3131                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3132
3133                                       /* op1:   0 == array, 1 == class                */
3134                                       /* val.a: (classinfo*) superclass               */
3135
3136 /*          superclass is an interface:
3137  *
3138  *          return (sub != NULL) &&
3139  *                 (sub->vftbl->interfacetablelength > super->index) &&
3140  *                 (sub->vftbl->interfacetable[-super->index] != NULL);
3141  *
3142  *          superclass is a class:
3143  *
3144  *          return ((sub != NULL) && (0
3145  *                  <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3146  *                  super->vftbl->diffvall));
3147  */
3148
3149                         {
3150                         classinfo *super = (classinfo*) iptr->val.a;
3151
3152 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3153             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3154 #endif
3155
3156                         var_to_reg_int(s1, src, REG_ITMP1);
3157                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3158                         if (s1 == d) {
3159                                 M_INTMOVE(s1, REG_ITMP1);
3160                                 s1 = REG_ITMP1;
3161                         }
3162                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3163                         if (iptr->op1) {                               /* class/interface */
3164                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3165                                         x86_64_test_reg_reg(cd, s1, s1);
3166
3167                                         /* TODO: clean up this calculation */
3168                                         a = 3;    /* mov_membase_reg */
3169                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3170
3171                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3172                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3173
3174                                         a += 3;    /* sub */
3175                                         CALCIMMEDIATEBYTES(a, super->index);
3176
3177                                         a += 3;    /* test */
3178
3179                                         a += 6;    /* jcc */
3180                                         a += 3;    /* mov_membase_reg */
3181                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3182
3183                                         a += 3;    /* test */
3184                                         a += 4;    /* setcc */
3185
3186                                         x86_64_jcc(cd, X86_64_CC_E, a);
3187
3188                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3189                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3190                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3191                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3192
3193                                         /* TODO: clean up this calculation */
3194                                         a = 0;
3195                                         a += 3;    /* mov_membase_reg */
3196                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3197
3198                                         a += 3;    /* test */
3199                                         a += 4;    /* setcc */
3200
3201                                         x86_64_jcc(cd, X86_64_CC_LE, a);
3202                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP1);
3203                                         x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3204                                         x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3205
3206                                 } else {                                   /* class           */
3207                                         x86_64_test_reg_reg(cd, s1, s1);
3208
3209                                         /* TODO: clean up this calculation */
3210                                         a = 3;    /* mov_membase_reg */
3211                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3212
3213                                         a += 10;   /* mov_imm_reg */
3214
3215                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3216                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3217
3218                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3219                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3220
3221                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3222                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3223
3224                                         a += 3;    /* sub */
3225                                         a += 3;    /* xor */
3226                                         a += 3;    /* cmp */
3227                                         a += 4;    /* setcc */
3228
3229                                         x86_64_jcc(cd, X86_64_CC_E, a);
3230
3231                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3232                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3233 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3234                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3235 #endif
3236                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3237                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3238                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3239 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3240                     codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3241 #endif
3242                                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3243                                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3244                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3245                                         x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3246                                 }
3247                         }
3248                         else
3249                                 panic("internal error: no inlined array instanceof");
3250                         }
3251                         store_reg_to_var_int(iptr->dst, d);
3252                         break;
3253
3254                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3255
3256                                       /* op1:   0 == array, 1 == class                */
3257                                       /* val.a: (classinfo*) superclass               */
3258
3259 /*          superclass is an interface:
3260  *
3261  *          OK if ((sub == NULL) ||
3262  *                 (sub->vftbl->interfacetablelength > super->index) &&
3263  *                 (sub->vftbl->interfacetable[-super->index] != NULL));
3264  *
3265  *          superclass is a class:
3266  *
3267  *          OK if ((sub == NULL) || (0
3268  *                 <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3269  *                 super->vftbl->diffvall));
3270  */
3271
3272                         {
3273                         classinfo *super = (classinfo*) iptr->val.a;
3274
3275 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3276             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3277 #endif
3278                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3279                         var_to_reg_int(s1, src, d);
3280                         if (iptr->op1) {                               /* class/interface */
3281                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3282                                         x86_64_test_reg_reg(cd, s1, s1);
3283
3284                                         /* TODO: clean up this calculation */
3285                                         a = 3;    /* mov_membase_reg */
3286                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3287
3288                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3289                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3290
3291                                         a += 3;    /* sub */
3292                                         CALCIMMEDIATEBYTES(a, super->index);
3293
3294                                         a += 3;    /* test */
3295                                         a += 6;    /* jcc */
3296
3297                                         a += 3;    /* mov_membase_reg */
3298                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3299
3300                                         a += 3;    /* test */
3301                                         a += 6;    /* jcc */
3302
3303                                         x86_64_jcc(cd, X86_64_CC_E, a);
3304
3305                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3306                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3307                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3308                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3309                                         x86_64_jcc(cd, X86_64_CC_LE, 0);
3310                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3311                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP2);
3312                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3313                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3314                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3315
3316                                 } else {                                     /* class           */
3317                                         x86_64_test_reg_reg(cd, s1, s1);
3318
3319                                         /* TODO: clean up this calculation */
3320                                         a = 3;    /* mov_membase_reg */
3321                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3322                                         a += 10;   /* mov_imm_reg */
3323                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3324                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3325
3326                                         if (d != REG_ITMP3) {
3327                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3328                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3329                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3330                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3331                                                 a += 3;    /* sub */
3332
3333                                         } else {
3334                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3335                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3336                                                 a += 3;    /* sub */
3337                                                 a += 10;   /* mov_imm_reg */
3338                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3339                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3340                                         }
3341
3342                                         a += 3;    /* cmp */
3343                                         a += 6;    /* jcc */
3344
3345                                         x86_64_jcc(cd, X86_64_CC_E, a);
3346
3347                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3348                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3349 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3350                     codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3351 #endif
3352                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3353                                         if (d != REG_ITMP3) {
3354                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3355                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3356 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3357                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3358 #endif
3359                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3360
3361                                         } else {
3362                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP2);
3363                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3364                                                 x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3365                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3366 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3367                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3368 #endif
3369                                         }
3370                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3371                                         x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3372                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3373                                 }
3374
3375                         } else
3376                                 panic("internal error: no inlined array checkcast");
3377                         }
3378                         M_INTMOVE(s1, d);
3379                         store_reg_to_var_int(iptr->dst, d);
3380                         break;
3381
3382                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3383
3384                         if (src->flags & INMEMORY) {
3385                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3386
3387                         } else {
3388                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3389                         }
3390                         x86_64_jcc(cd, X86_64_CC_L, 0);
3391                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3392                         break;
3393
3394                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3395
3396                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3397                         x86_64_jcc(cd, X86_64_CC_E, 0);
3398                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3399                         break;
3400
3401                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3402                                          /* op1 = dimension, val.a = array descriptor */
3403
3404                         /* check for negative sizes and copy sizes to stack if necessary  */
3405
3406                         MCODECHECK((iptr->op1 << 1) + 64);
3407
3408                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3409                                 var_to_reg_int(s2, src, REG_ITMP1);
3410                                 x86_64_testl_reg_reg(cd, s2, s2);
3411                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3412                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3413
3414                                 /* copy sizes to stack (argument numbers >= INT_ARG_CNT)      */
3415
3416                                 if (src->varkind != ARGVAR) {
3417                                         x86_64_mov_reg_membase(cd, s2, REG_SP, (s1 + INT_ARG_CNT) * 8);
3418                                 }
3419                         }
3420
3421                         /* a0 = dimension count */
3422                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3423
3424                         /* a1 = arraydescriptor */
3425                         x86_64_mov_imm_reg(cd, (s8) iptr->val.a, rd->argintregs[1]);
3426
3427                         /* a2 = pointer to dimensions = stack pointer */
3428                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3429
3430                         x86_64_mov_imm_reg(cd, (s8) builtin_nmultianewarray, REG_ITMP1);
3431                         x86_64_call_reg(cd, REG_ITMP1);
3432
3433                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3434                         M_INTMOVE(REG_RESULT, s1);
3435                         store_reg_to_var_int(iptr->dst, s1);
3436                         break;
3437
3438                 default:
3439                         throw_cacao_exception_exit(string_java_lang_InternalError,
3440                                                                            "Unknown ICMD %d", iptr->opc);
3441         } /* switch */
3442
3443         } /* for instruction */
3444
3445         /* copy values to interface registers */
3446
3447         src = bptr->outstack;
3448         len = bptr->outdepth;
3449         MCODECHECK(64 + len);
3450         while (src) {
3451                 len--;
3452                 if ((src->varkind != STACKVAR)) {
3453                         s2 = src->type;
3454                         if (IS_FLT_DBL_TYPE(s2)) {
3455                                 var_to_reg_flt(s1, src, REG_FTMP1);
3456                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3457                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3458
3459                                 } else {
3460                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3461                                 }
3462
3463                         } else {
3464                                 var_to_reg_int(s1, src, REG_ITMP1);
3465                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3466                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3467
3468                                 } else {
3469                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3470                                 }
3471                         }
3472                 }
3473                 src = src->prev;
3474         }
3475         } /* if (bptr -> flags >= BBREACHED) */
3476         } /* for basic block */
3477
3478         {
3479
3480         /* generate bound check stubs */
3481
3482         u1 *xcodeptr = NULL;
3483         branchref *bref;
3484
3485         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3486                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3487                                   bref->branchpos,
3488                                                   cd->mcodeptr - cd->mcodebase);
3489
3490                 MCODECHECK(50);
3491
3492                 /* move index register into REG_ITMP1 */
3493                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);              /* 3 bytes  */
3494
3495                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                  /* 10 bytes */
3496                 dseg_adddata(cd, cd->mcodeptr);
3497                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3498                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3499
3500                 if (xcodeptr != NULL) {
3501                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3502
3503                 } else {
3504                         xcodeptr = cd->mcodeptr;
3505
3506                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3507                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3508
3509                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3510                         x86_64_mov_imm_reg(cd, (s8) new_arrayindexoutofboundsexception, REG_ITMP3);
3511                         x86_64_call_reg(cd, REG_ITMP3);
3512
3513                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3514                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3515
3516                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3517                         x86_64_jmp_reg(cd, REG_ITMP3);
3518                 }
3519         }
3520
3521         /* generate negative array size check stubs */
3522
3523         xcodeptr = NULL;
3524
3525         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3526                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3527                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3528                                                           bref->branchpos,
3529                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3530                         continue;
3531                 }
3532
3533                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3534                                   bref->branchpos,
3535                                                   cd->mcodeptr - cd->mcodebase);
3536
3537                 MCODECHECK(50);
3538
3539                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                         /* 10 bytes */
3540                 dseg_adddata(cd, cd->mcodeptr);
3541                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3542                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);     /* 3 bytes  */
3543
3544                 if (xcodeptr != NULL) {
3545                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3546
3547                 } else {
3548                         xcodeptr = cd->mcodeptr;
3549
3550                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3551                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3552
3553                         x86_64_mov_imm_reg(cd, (s8) new_negativearraysizeexception, REG_ITMP3);
3554                         x86_64_call_reg(cd, REG_ITMP3);
3555
3556                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3557                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3558
3559                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3560                         x86_64_jmp_reg(cd, REG_ITMP3);
3561                 }
3562         }
3563
3564         /* generate cast check stubs */
3565
3566         xcodeptr = NULL;
3567
3568         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3569                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3570                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3571                                                           bref->branchpos,
3572                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3573                         continue;
3574                 }
3575
3576                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3577                                   bref->branchpos,
3578                                                   cd->mcodeptr - cd->mcodebase);
3579
3580                 MCODECHECK(50);
3581
3582                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3583                 dseg_adddata(cd, cd->mcodeptr);
3584                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);     /* 10 bytes */
3585                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3586
3587                 if (xcodeptr != NULL) {
3588                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3589
3590                 } else {
3591                         xcodeptr = cd->mcodeptr;
3592
3593                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3594                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3595
3596                         x86_64_mov_imm_reg(cd, (s8) new_classcastexception, REG_ITMP3);
3597                         x86_64_call_reg(cd, REG_ITMP3);
3598
3599                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3600                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3601
3602                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3603                         x86_64_jmp_reg(cd, REG_ITMP3);
3604                 }
3605         }
3606
3607         /* generate divide by zero check stubs */
3608
3609         xcodeptr = NULL;
3610
3611         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3612                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3613                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3614                                                           bref->branchpos,
3615                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3616                         continue;
3617                 }
3618
3619                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3620                                   bref->branchpos,
3621                                                   cd->mcodeptr - cd->mcodebase);
3622
3623                 MCODECHECK(50);
3624
3625                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3626                 dseg_adddata(cd, cd->mcodeptr);
3627                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);      /* 10 bytes */
3628                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3629
3630                 if (xcodeptr != NULL) {
3631                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3632
3633                 } else {
3634                         xcodeptr = cd->mcodeptr;
3635
3636                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3637                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3638
3639                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
3640                         x86_64_call_reg(cd, REG_ITMP3);
3641
3642                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3643                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3644
3645                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3646                         x86_64_jmp_reg(cd, REG_ITMP3);
3647                 }
3648         }
3649
3650         /* generate exception check stubs */
3651
3652         xcodeptr = NULL;
3653
3654         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3655                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3656                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3657                                                           bref->branchpos,
3658                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3659                         continue;
3660                 }
3661
3662                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3663                                   bref->branchpos,
3664                                                   cd->mcodeptr - cd->mcodebase);
3665
3666                 MCODECHECK(50);
3667
3668                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3669                 dseg_adddata(cd, cd->mcodeptr);
3670                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3671                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3672
3673                 if (xcodeptr != NULL) {
3674                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3675
3676                 } else {
3677                         xcodeptr = cd->mcodeptr;
3678
3679 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3680                         x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3681                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0);
3682                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3683                         x86_64_call_reg(cd, REG_ITMP1);
3684                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3685                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3686                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3687                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
3688                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
3689 #else
3690                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
3691                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
3692                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
3693 #endif
3694
3695                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3696                         x86_64_jmp_reg(cd, REG_ITMP3);
3697                 }
3698         }
3699
3700         /* generate null pointer check stubs */
3701
3702         xcodeptr = NULL;
3703
3704         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3705                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3706                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3707                                                           bref->branchpos,
3708                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3709                         continue;
3710                 }
3711
3712                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3713                                   bref->branchpos,
3714                                                   cd->mcodeptr - cd->mcodebase);
3715
3716                 MCODECHECK(50);
3717
3718                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3719                 dseg_adddata(cd, cd->mcodeptr);
3720                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3721                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3722
3723                 if (xcodeptr != NULL) {
3724                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3725
3726                 } else {
3727                         xcodeptr = cd->mcodeptr;
3728
3729                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3730                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3731
3732                         x86_64_mov_imm_reg(cd, (s8) new_nullpointerexception, REG_ITMP3);
3733                         x86_64_call_reg(cd, REG_ITMP3);
3734
3735                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3736                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3737
3738                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3739                         x86_64_jmp_reg(cd, REG_ITMP3);
3740                 }
3741         }
3742
3743         /* generate put/getstatic stub call code */
3744
3745         {
3746                 clinitref   *cref;
3747                 codegendata *tmpcd;
3748                 u1           xmcode;
3749                 u4           mcode;
3750
3751                 tmpcd = DNEW(codegendata);
3752
3753                 for (cref = cd->clinitrefs; cref != NULL; cref = cref->next) {
3754                         /* Get machine code which is patched back in later. A             */
3755                         /* `call rel32' is 5 bytes long.                                  */
3756                         xcodeptr = cd->mcodebase + cref->branchpos;
3757                         xmcode = *xcodeptr;
3758                         mcode = *((u4 *) (xcodeptr + 1));
3759
3760                         MCODECHECK(50);
3761
3762                         /* patch in `call rel32' to call the following code               */
3763                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
3764                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
3765
3766                         /* Save current stack pointer into a temporary register.          */
3767                         x86_64_mov_reg_reg(cd, REG_SP, REG_ITMP1);
3768
3769                         /* Push machine code bytes to patch onto the stack.               */
3770                         x86_64_push_imm(cd, (u1) xmcode);
3771                         x86_64_push_imm(cd, (u4) mcode);
3772
3773                         x86_64_push_imm(cd, (u8) cref->class);
3774
3775                         /* Push previously saved stack pointer onto stack.                */
3776                         x86_64_push_reg(cd, REG_ITMP1);
3777
3778                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP1);
3779                         x86_64_jmp_reg(cd, REG_ITMP1);
3780                 }
3781         }
3782         }
3783
3784         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
3785 }
3786
3787
3788 /* function createcompilerstub *************************************************
3789
3790         creates a stub routine which calls the compiler
3791
3792 *******************************************************************************/
3793
3794 #define COMPSTUBSIZE 23
3795
3796 u1 *createcompilerstub(methodinfo *m)
3797 {
3798         u1 *s = CNEW(u1, COMPSTUBSIZE);     /* memory to hold the stub            */
3799         codegendata *cd;
3800         s4 dumpsize;
3801
3802         /* mark start of dump memory area */
3803
3804         dumpsize = dump_size();
3805
3806         cd = DNEW(codegendata);
3807         cd->mcodeptr = s;
3808
3809         /* code for the stub */
3810         x86_64_mov_imm_reg(cd, (u8) m, REG_ITMP1); /* pass method to compiler     */
3811         x86_64_mov_imm_reg(cd, (u8) asm_call_jit_compiler, REG_ITMP3);/* load address */
3812         x86_64_jmp_reg(cd, REG_ITMP3);      /* jump to compiler                   */
3813
3814 #if defined(STATISTICS)
3815         if (opt_stat)
3816                 count_cstub_len += COMPSTUBSIZE;
3817 #endif
3818
3819         /* release dump area */
3820
3821         dump_release(dumpsize);
3822
3823         return s;
3824 }
3825
3826
3827 /* function removecompilerstub *************************************************
3828
3829      deletes a compilerstub from memory  (simply by freeing it)
3830
3831 *******************************************************************************/
3832
3833 void removecompilerstub(u1 *stub)
3834 {
3835         CFREE(stub, COMPSTUBSIZE);
3836 }
3837
3838
3839 /* function: createnativestub **************************************************
3840
3841         creates a stub routine which calls a native method
3842
3843 *******************************************************************************/
3844
3845 /* #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3846 /* static java_objectheader **(*callgetexceptionptrptr)() = builtin_get_exceptionptrptr; */
3847 /* #endif */
3848
3849 #define NATIVESTUBSIZE 420
3850
3851 u1 *createnativestub(functionptr f, methodinfo *m)
3852 {
3853         u1 *s = CNEW(u1, NATIVESTUBSIZE);   /* memory to hold the stub            */
3854         s4 stackframesize;                  /* size of stackframe if needed       */
3855         codegendata *cd;
3856         registerdata *rd;
3857         t_inlining_globals *id;
3858         s4 dumpsize;
3859
3860         void **callAddrPatchPos=0;
3861         u1 *jmpInstrPos=0;
3862         void **jmpInstrPatchPos=0;
3863
3864         /* mark start of dump memory area */
3865
3866         dumpsize = dump_size();
3867
3868         cd = DNEW(codegendata);
3869         rd = DNEW(registerdata);
3870         id = DNEW(t_inlining_globals);
3871
3872         /* setup registers before using it */
3873
3874         inlining_setup(m, id);
3875         reg_setup(m, rd, id);
3876
3877         /* set some required varibles which are normally set by codegen_setup */
3878         cd->mcodebase = s;
3879         cd->mcodeptr = s;
3880         cd->clinitrefs = NULL;
3881
3882     descriptor2types(m);                /* set paramcount and paramtypes      */
3883
3884         /* if function is static, check for initialized */
3885
3886         if (m->flags & ACC_STATIC) {
3887                 /* if class isn't yet initialized, do it */
3888                 if (!m->class->initialized) {
3889                         codegen_addclinitref(cd, cd->mcodeptr, m->class);
3890                 }
3891         }
3892
3893
3894         x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3895
3896         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
3897         x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
3898         x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
3899         x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
3900         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
3901         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
3902
3903         x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
3904         x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
3905         x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
3906         x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
3907 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8); */
3908 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8); */
3909 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8); */
3910 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8); */
3911
3912         if (runverbose) {
3913                 s4 p, l, s1;
3914
3915                 /* show integer hex code for float arguments */
3916                 for (p = 0, l = 0; p < m->paramcount; p++) {
3917                         if (IS_FLT_DBL_TYPE(m->paramtypes[p])) {
3918                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
3919                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
3920                                 }
3921
3922                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
3923                                 l++;
3924                         }
3925                 }
3926
3927                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1);
3928                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
3929                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
3930                 x86_64_call_reg(cd, REG_ITMP1);
3931         }
3932 /* call method to resolve native function if needed */
3933 #ifndef STATIC_CLASSPATH
3934         if (f==0) { /* only if not already resolved */
3935                 x86_64_jmp_imm(cd,0);
3936                 jmpInstrPos=cd->mcodeptr-4; /*needed to patch a jump over this block*/
3937                 x86_64_mov_imm_reg(cd,(u8)m,rd->argintregs[0]);
3938                 x86_64_mov_imm_reg(cd,0,rd->argintregs[1]);
3939                 callAddrPatchPos=cd->mcodeptr-8; /* at this position the place is specified where the native function adress should be patched into*/
3940                 x86_64_mov_imm_reg(cd,0,rd->argintregs[2]);
3941                 jmpInstrPatchPos=cd->mcodeptr-8;
3942                 x86_64_mov_imm_reg(cd,jmpInstrPos,rd->argintregs[3]);
3943                 x86_64_mov_imm_reg(cd,(s8)codegen_resolve_native,REG_ITMP1);
3944                 x86_64_call_reg(cd,REG_ITMP1);
3945                 *(jmpInstrPatchPos)=cd->mcodeptr-jmpInstrPos-1; /*=opcode jmp_imm size*/
3946         }
3947 #endif
3948
3949         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
3950         x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
3951         x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
3952         x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
3953         x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
3954         x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
3955
3956         x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
3957         x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
3958         x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
3959         x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
3960 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]); */
3961 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]); */
3962 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]); */
3963 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]); */
3964
3965         x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3966
3967
3968 #if 0
3969         x86_64_alu_imm_reg(cd, X86_64_SUB, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3970
3971         /* save callee saved float registers */
3972         x86_64_movq_reg_membase(cd, XMM15, REG_SP, 0 * 8);
3973         x86_64_movq_reg_membase(cd, XMM14, REG_SP, 1 * 8);
3974         x86_64_movq_reg_membase(cd, XMM13, REG_SP, 2 * 8);
3975         x86_64_movq_reg_membase(cd, XMM12, REG_SP, 3 * 8);
3976         x86_64_movq_reg_membase(cd, XMM11, REG_SP, 4 * 8);
3977         x86_64_movq_reg_membase(cd, XMM10, REG_SP, 5 * 8);
3978 #endif
3979
3980         /* save argument registers on stack -- if we have to */
3981         if ((m->flags & ACC_STATIC && m->paramcount > (INT_ARG_CNT - 2)) || m->paramcount > (INT_ARG_CNT - 1)) {
3982                 s4 i;
3983                 s4 paramshiftcnt = (m->flags & ACC_STATIC) ? 2 : 1;
3984                 s4 stackparamcnt = (m->paramcount > INT_ARG_CNT) ? m->paramcount - INT_ARG_CNT : 0;
3985
3986                 stackframesize = stackparamcnt + paramshiftcnt;
3987
3988                 /* keep stack 16-byte aligned */
3989                 if (!(stackframesize & 0x1))
3990                         stackframesize++;
3991
3992                 x86_64_alu_imm_reg(cd, X86_64_SUB, stackframesize * 8, REG_SP);
3993
3994                 /* copy stack arguments into new stack frame -- if any */
3995                 for (i = 0; i < stackparamcnt; i++) {
3996                         x86_64_mov_membase_reg(cd, REG_SP, (stackparamcnt + 1 + i) * 8, REG_ITMP1);
3997                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, (paramshiftcnt + i) * 8);
3998                 }
3999
4000                 if (m->flags & ACC_STATIC) {
4001                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 1 * 8);
4002                         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 0 * 8);
4003
4004                 } else {
4005                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 0 * 8);
4006                 }
4007
4008         } else {
4009                 /* keep stack 16-byte aligned */
4010                 x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
4011                 stackframesize = 1;
4012         }
4013
4014         if (m->flags & ACC_STATIC) {
4015                 x86_64_mov_reg_reg(cd, rd->argintregs[3], rd->argintregs[5]);
4016                 x86_64_mov_reg_reg(cd, rd->argintregs[2], rd->argintregs[4]);
4017                 x86_64_mov_reg_reg(cd, rd->argintregs[1], rd->argintregs[3]);
4018                 x86_64_mov_reg_reg(cd, rd->argintregs[0], rd->argintregs[2]);
4019
4020                 /* put class into second argument register */
4021                 x86_64_mov_imm_reg(cd, (u8) m->class, rd->argintregs[1]);
4022
4023         } else {
4024                 x86_64_mov_reg_reg(cd, rd->argintregs[4], rd->argintregs[5]);
4025                 x86_64_mov_reg_reg(cd, rd->argintregs[3], rd->argintregs[4]);
4026                 x86_64_mov_reg_reg(cd, rd->argintregs[2], rd->argintregs[3]);
4027                 x86_64_mov_reg_reg(cd, rd->argintregs[1], rd->argintregs[2]);
4028                 x86_64_mov_reg_reg(cd, rd->argintregs[0], rd->argintregs[1]);
4029         }
4030
4031         /* put env into first argument register */
4032         x86_64_mov_imm_reg(cd, (u8) &env, rd->argintregs[0]);
4033
4034         x86_64_mov_imm_reg(cd, (u8) f, REG_ITMP1);
4035 #ifndef STATIC_CLASSPATH
4036         if (f==0)
4037                 (*callAddrPatchPos)=cd->mcodeptr-8;
4038 #endif
4039         x86_64_call_reg(cd, REG_ITMP1);
4040
4041         /* remove stackframe if there is one */
4042         if (stackframesize) {
4043                 x86_64_alu_imm_reg(cd, X86_64_ADD, stackframesize * 8, REG_SP);
4044         }
4045
4046         if (runverbose) {
4047                 x86_64_alu_imm_reg(cd, X86_64_SUB, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
4048
4049                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
4050                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
4051
4052                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
4053                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
4054                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4055                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4056
4057                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
4058                 x86_64_call_reg(cd, REG_ITMP1);
4059
4060                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
4061                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
4062
4063                 x86_64_alu_imm_reg(cd, X86_64_ADD, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
4064         }
4065
4066 #if 0
4067         /* restore callee saved registers */
4068         x86_64_movq_membase_reg(cd, REG_SP, 0 * 8, XMM15);
4069         x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, XMM14);
4070         x86_64_movq_membase_reg(cd, REG_SP, 2 * 8, XMM13);
4071         x86_64_movq_membase_reg(cd, REG_SP, 3 * 8, XMM12);
4072         x86_64_movq_membase_reg(cd, REG_SP, 4 * 8, XMM11);
4073         x86_64_movq_membase_reg(cd, REG_SP, 5 * 8, XMM10);
4074
4075         x86_64_alu_imm_reg(cd, X86_64_ADD, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
4076 #endif
4077
4078 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4079         x86_64_push_reg(cd, REG_RESULT);
4080 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4081         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4082         x86_64_call_reg(cd, REG_ITMP3);
4083         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
4084         x86_64_pop_reg(cd, REG_RESULT);
4085 #else
4086         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
4087         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
4088 #endif
4089         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
4090         x86_64_jcc(cd, X86_64_CC_NE, 1);
4091
4092         x86_64_ret(cd);
4093
4094 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4095         x86_64_push_reg(cd, REG_ITMP3);
4096 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4097         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4098         x86_64_call_reg(cd, REG_ITMP3);
4099         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4100         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
4101 #else
4102         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4103         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
4104         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4105         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4106 #endif
4107
4108         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
4109         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
4110
4111         x86_64_mov_imm_reg(cd, (s8) asm_handle_nat_exception, REG_ITMP3);
4112         x86_64_jmp_reg(cd, REG_ITMP3);
4113
4114         {
4115                 u1          *xcodeptr;
4116                 clinitref   *cref;
4117                 codegendata *tmpcd;
4118                 u1           xmcode;
4119                 u4           mcode;
4120
4121                 tmpcd = DNEW(codegendata);
4122
4123                 /* there can only be one clinit ref entry                             */
4124                 cref = cd->clinitrefs;
4125
4126                 if (cref) {
4127                         /* Get machine code which is patched back in later. A             */
4128                         /* `call rel32' is 5 bytes long.                                  */
4129                         xcodeptr = cd->mcodebase + cref->branchpos;
4130                         xmcode = *xcodeptr;
4131                         mcode = *((u4 *) (xcodeptr + 1));
4132
4133                         /* patch in `call rel32' to call the following code               */
4134                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4135                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4136
4137                         /* Save current stack pointer into a temporary register.          */
4138                         x86_64_mov_reg_reg(cd, REG_SP, REG_ITMP1);
4139
4140                         /* Push machine code bytes to patch onto the stack.               */
4141                         x86_64_push_imm(cd, (u1) xmcode);
4142                         x86_64_push_imm(cd, (u4) mcode);
4143
4144                         x86_64_push_imm(cd, (u8) cref->class);
4145
4146                         /* Push previously saved stack pointer onto stack.                */
4147                         x86_64_push_reg(cd, REG_ITMP1);
4148
4149                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP1);
4150                         x86_64_jmp_reg(cd, REG_ITMP1);
4151                 }
4152         }
4153
4154 #if 0
4155         {
4156                 static int stubprinted;
4157                 if (!stubprinted)
4158                         printf("stubsize: %d\n", ((long) cd->mcodeptr - (long) s));
4159                 stubprinted = 1;
4160         }
4161 #endif
4162
4163 #if defined(STATISTICS)
4164         if (opt_stat)
4165                 count_nstub_len += NATIVESTUBSIZE;
4166 #endif
4167
4168         /* release dump area */
4169
4170         dump_release(dumpsize);
4171
4172         return s;
4173 }
4174
4175
4176 /* function: removenativestub **************************************************
4177
4178     removes a previously created native-stub from memory
4179
4180 *******************************************************************************/
4181
4182 void removenativestub(u1 *stub)
4183 {
4184         CFREE(stub, NATIVESTUBSIZE);
4185 }
4186
4187
4188 /*
4189  * These are local overrides for various environment variables in Emacs.
4190  * Please do not remove this and leave it at the end of the file, where
4191  * Emacs will automagically detect them.
4192  * ---------------------------------------------------------------------
4193  * Local variables:
4194  * mode: c
4195  * indent-tabs-mode: t
4196  * c-basic-offset: 4
4197  * tab-width: 4
4198  * End:
4199  */