src/vm/jit/x86_64/codegen.c

   1 /* vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    $Id: codegen.c 1757 2004-12-13 16:58:37Z twisti $
  31
  32 */
  33
  34
  35 #define _GNU_SOURCE
  36
  37 #include <stdio.h>
  38 #include <ucontext.h>
  39
  40 #include "native/native.h"
  41 #include "vm/global.h"
  42 #include "vm/builtin.h"
  43 #include "vm/loader.h"
  44 #include "vm/tables.h"
  45 #include "vm/jit/asmpart.h"
  46 #include "vm/jit/jit.h"
  47 #include "vm/jit/reg.h"
  48 #include "vm/jit/parse.h"
  49 #include "vm/jit/x86_64/arch.h"
  50 #include "vm/jit/x86_64/codegen.h"
  51 #include "vm/jit/x86_64/emitfuncs.h"
  52 #include "vm/jit/x86_64/types.h"
  53 #include "vm/jit/x86_64/asmoffsets.h"
  54
  55
  56 /* register descripton - array ************************************************/
  57
  58 /* #define REG_RES   0         reserved register for OS or code generator     */
  59 /* #define REG_RET   1         return value register                          */
  60 /* #define REG_EXC   2         exception value register (only old jit)        */
  61 /* #define REG_SAV   3         (callee) saved register                        */
  62 /* #define REG_TMP   4         scratch temporary register (caller saved)      */
  63 /* #define REG_ARG   5         argument register (caller saved)               */
  64
  65 /* #define REG_END   -1        last entry in tables                           */
  66
  67 static int nregdescint[] = {
  68     REG_RET, REG_ARG, REG_ARG, REG_TMP, REG_RES, REG_SAV, REG_ARG, REG_ARG,
  69     REG_ARG, REG_ARG, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV,
  70     REG_END
  71 };
  72
  73
  74 static int nregdescfloat[] = {
  75     REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG,
  76     REG_RES, REG_RES, REG_RES, REG_TMP, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  77     REG_END
  78 };
  79
  80
  81 /* Include independent code generation stuff -- include after register        */
  82 /* descriptions to avoid extern definitions.                                  */
  83
  84 #include "vm/jit/codegen.inc"
  85 #include "vm/jit/reg.inc"
  86 #ifdef LSRA
  87 #include "vm/jit/lsra.inc"
  88 #endif
  89
  90
  91 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
  92 void thread_restartcriticalsection(ucontext_t *uc)
  93 {
  94         void *critical;
  95
  96         critical = thread_checkcritical((void *) uc->uc_mcontext.gregs[REG_RIP]);
  97
  98         if (critical)
  99                 uc->uc_mcontext.gregs[REG_RIP] = (u8) critical;
 100 }
 101 #endif
 102
 103
 104 /* NullPointerException signal handler for hardware null pointer check */
 105
 106 void catch_NullPointerException(int sig, siginfo_t *siginfo, void *_p)
 107 {
 108         sigset_t nsig;
 109         /*      int      instr; */
 110         /*      long     faultaddr; */
 111
 112         struct ucontext *_uc = (struct ucontext *) _p;
 113         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 114         struct sigaction act;
 115         java_objectheader *xptr;
 116
 117         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 118
 119
 120 /*      instr = *((int*)(sigctx->rip)); */
 121 /*      faultaddr = sigctx->sc_regs[(instr >> 16) & 0x1f]; */
 122
 123 /*      if (faultaddr == 0) { */
 124         act.sa_sigaction = (functionptr) catch_NullPointerException; /* reinstall handler */
 125         act.sa_flags = SA_SIGINFO;
 126         sigaction(sig, &act, NULL);
 127
 128         sigemptyset(&nsig);
 129         sigaddset(&nsig, sig);
 130         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 131
 132         xptr = new_nullpointerexception();
 133
 134         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 135         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 136         sigctx->rip = (u8) asm_handle_exception;
 137
 138         return;
 139
 140 /*      } else { */
 141 /*              faultaddr += (long) ((instr << 16) >> 16); */
 142 /*              fprintf(stderr, "faulting address: 0x%08x\n", faultaddr); */
 143 /*              panic("Stack overflow"); */
 144 /*      } */
 145 }
 146
 147
 148 /* ArithmeticException signal handler for hardware divide by zero check */
 149
 150 void catch_ArithmeticException(int sig, siginfo_t *siginfo, void *_p)
 151 {
 152         sigset_t nsig;
 153
 154         struct ucontext *_uc = (struct ucontext *) _p;
 155         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 156         struct sigaction act;
 157         java_objectheader *xptr;
 158
 159         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 160
 161         act.sa_sigaction = (functionptr) catch_ArithmeticException; /* reinstall handler */
 162         act.sa_flags = SA_SIGINFO;
 163         sigaction(sig, &act, NULL);
 164
 165         sigemptyset(&nsig);
 166         sigaddset(&nsig, sig);
 167         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 168
 169         xptr = new_arithmeticexception();
 170
 171         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 172         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 173         sigctx->rip = (u8) asm_handle_exception;
 174
 175         return;
 176 }
 177
 178
 179 void init_exceptions(void)
 180 {
 181         struct sigaction act;
 182
 183         /* install signal handlers we need to convert to exceptions */
 184         sigemptyset(&act.sa_mask);
 185
 186         if (!checknull) {
 187 #if defined(SIGSEGV)
 188                 act.sa_sigaction = (functionptr) catch_NullPointerException;
 189                 act.sa_flags = SA_SIGINFO;
 190                 sigaction(SIGSEGV, &act, NULL);
 191 #endif
 192
 193 #if defined(SIGBUS)
 194                 act.sa_sigaction = (functionptr) catch_NullPointerException;
 195                 act.sa_flags = SA_SIGINFO;
 196                 sigaction(SIGBUS, &act, NULL);
 197 #endif
 198         }
 199
 200         act.sa_sigaction = (functionptr) catch_ArithmeticException;
 201         act.sa_flags = SA_SIGINFO;
 202         sigaction(SIGFPE, &act, NULL);
 203 }
 204
 205
 206 /* function gen_mcode **********************************************************
 207
 208         generates machine code
 209
 210 *******************************************************************************/
 211
 212 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
 213 {
 214         s4 len, s1, s2, s3, d;
 215         s8 a;
 216         s4 parentargs_base;
 217         stackptr        src;
 218         varinfo        *var;
 219         basicblock     *bptr;
 220         instruction    *iptr;
 221         exceptiontable *ex;
 222
 223         {
 224         s4 i, p, pa, t, l;
 225         s4 savedregs_num;
 226
 227         savedregs_num = 0;
 228
 229         /* space to save used callee saved registers */
 230
 231         savedregs_num += (rd->savintregcnt - rd->maxsavintreguse);
 232         savedregs_num += (rd->savfltregcnt - rd->maxsavfltreguse);
 233
 234         parentargs_base = rd->maxmemuse + savedregs_num;
 235
 236 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 237
 238         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 239                 parentargs_base++;
 240
 241 #endif
 242
 243     /* keep stack 16-byte aligned for calls into native code e.g. libc or jni */
 244     /* (alignment problems with movaps)                                       */
 245
 246         if (!(parentargs_base & 0x1)) {
 247                 parentargs_base++;
 248         }
 249
 250         /* create method header */
 251
 252         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 253         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 254
 255 #if defined(USE_THREADS)
 256
 257         /* IsSync contains the offset relative to the stack pointer for the
 258            argument of monitor_exit used in the exception handler. Since the
 259            offset could be zero and give a wrong meaning of the flag it is
 260            offset by one.
 261         */
 262
 263         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 264                 (void) dseg_adds4(cd, (rd->maxmemuse + 1) * 8);     /* IsSync         */
 265         else
 266
 267 #endif
 268
 269                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 270
 271         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 272         (void) dseg_adds4(cd, rd->savintregcnt - rd->maxsavintreguse);/* IntSave  */
 273         (void) dseg_adds4(cd, rd->savfltregcnt - rd->maxsavfltreguse);/* FltSave  */
 274         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 275
 276         /* create exception table */
 277
 278         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 279                 dseg_addtarget(cd, ex->start);
 280                 dseg_addtarget(cd, ex->end);
 281                 dseg_addtarget(cd, ex->handler);
 282                 (void) dseg_addaddress(cd, ex->catchtype);
 283         }
 284
 285         /* initialize mcode variables */
 286
 287         cd->mcodeptr = (u1 *) cd->mcodebase;
 288         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 289         MCODECHECK(128 + m->paramcount);
 290
 291         /* create stack frame (if necessary) */
 292
 293         if (parentargs_base) {
 294                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 295         }
 296
 297         /* save return address and used callee saved registers */
 298
 299         p = parentargs_base;
 300         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
 301                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 302         }
 303         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
 304                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 305         }
 306
 307         /* take arguments out of register or stack frame */
 308
 309         for (p = 0, l = 0, s1 = 0, s2 = 0; p < m->paramcount; p++) {
 310                 t = m->paramtypes[p];
 311                 var = &(rd->locals[l][t]);
 312                 l++;
 313                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 314                         l++;
 315                 if (var->type < 0) {
 316                         if (IS_INT_LNG_TYPE(t)) {
 317                                 s1++;
 318                         } else {
 319                                 s2++;
 320                         }
 321                         continue;
 322                 }
 323                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 324                         if (s1 < INT_ARG_CNT) {                  /* register arguments    */
 325                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 326                                         M_INTMOVE(rd->argintregs[s1], var->regoff);
 327
 328                                 } else {                             /* reg arg -> spilled    */
 329                                     x86_64_mov_reg_membase(cd, rd->argintregs[s1], REG_SP, var->regoff * 8);
 330                                 }
 331
 332                         } else {                                 /* stack arguments       */
 333                                 pa = s1 - INT_ARG_CNT;
 334                                 if (s2 >= FLT_ARG_CNT) {
 335                                         pa += s2 - FLT_ARG_CNT;
 336                                 }
 337                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 338                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);    /* + 8 for return address */
 339                                 } else {                             /* stack arg -> spilled  */
 340                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_ITMP1);    /* + 8 for return address */
 341                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, var->regoff * 8);
 342                                 }
 343                         }
 344                         s1++;
 345
 346                 } else {                                     /* floating args         */
 347                         if (s2 < FLT_ARG_CNT) {                  /* register arguments    */
 348                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 349                                         M_FLTMOVE(rd->argfltregs[s2], var->regoff);
 350
 351                                 } else {                                         /* reg arg -> spilled    */
 352                                         x86_64_movq_reg_membase(cd, rd->argfltregs[s2], REG_SP, var->regoff * 8);
 353                                 }
 354
 355                         } else {                                 /* stack arguments       */
 356                                 pa = s2 - FLT_ARG_CNT;
 357                                 if (s1 >= INT_ARG_CNT) {
 358                                         pa += s1 - INT_ARG_CNT;
 359                                 }
 360                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 361                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);
 362
 363                                 } else {
 364                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_FTMP1);
 365                                         x86_64_movq_reg_membase(cd, REG_FTMP1, REG_SP, var->regoff * 8);
 366                                 }
 367                         }
 368                         s2++;
 369                 }
 370         }  /* end for */
 371
 372         /* save monitorenter argument */
 373
 374 #if defined(USE_THREADS)
 375         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 376                 u8 func_enter;
 377
 378                 if (m->flags & ACC_STATIC) {
 379                         func_enter = (u8) builtin_staticmonitorenter;
 380                         x86_64_mov_imm_reg(cd, (s8) m->class, REG_ITMP1);
 381                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, rd->maxmemuse * 8);
 382
 383                 } else {
 384                         func_enter = (u8) builtin_monitorenter;
 385                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, rd->maxmemuse * 8);
 386                 }
 387
 388                 /* call monitorenter function */
 389
 390                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
 391                 x86_64_mov_imm_reg(cd, func_enter, REG_ITMP1);
 392                 x86_64_call_reg(cd, REG_ITMP1);
 393         }
 394 #endif
 395
 396         /* copy argument registers to stack and call trace function with pointer
 397            to arguments on stack.
 398         */
 399         if (runverbose) {
 400                 x86_64_alu_imm_reg(cd, X86_64_SUB, (6 + 8 + 1 + 1) * 8, REG_SP);
 401
 402                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
 403                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
 404                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
 405                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
 406                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
 407                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
 408
 409                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
 410                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
 411                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
 412                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
 413                 x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8);
 414                 x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8);
 415                 x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8);
 416                 x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8);
 417
 418                 /* show integer hex code for float arguments */
 419
 420                 for (p = 0, l = 0; p < m->paramcount; p++) {
 421                         t = m->paramtypes[p];
 422
 423                         if (IS_FLT_DBL_TYPE(t)) {
 424                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
 425                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
 426                                 }
 427
 428                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 429                                 l++;
 430                         }
 431                 }
 432
 433                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP2);
 434                 x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 435                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
 436                 x86_64_call_reg(cd, REG_ITMP1);
 437
 438                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
 439                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
 440                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
 441                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
 442                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
 443                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
 444
 445                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
 446                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
 447                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
 448                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
 449                 x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]);
 450                 x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]);
 451                 x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]);
 452                 x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]);
 453
 454                 x86_64_alu_imm_reg(cd, X86_64_ADD, (6 + 8 + 1 + 1) * 8, REG_SP);
 455         }
 456
 457         }
 458
 459         /* end of header generation */
 460
 461         /* walk through all basic blocks */
 462         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 463
 464                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 465
 466                 if (bptr->flags >= BBREACHED) {
 467
 468                         /* branch resolving */
 469
 470                         branchref *bref;
 471                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 472                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 473                                                                   bref->branchpos,
 474                                                                   bptr->mpc);
 475                         }
 476
 477                 /* copy interface registers to their destination */
 478
 479                 src = bptr->instack;
 480                 len = bptr->indepth;
 481                 MCODECHECK(64 + len);
 482                 while (src != NULL) {
 483                         len--;
 484                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 485                                 if (bptr->type == BBTYPE_SBR) {
 486                                         d = reg_of_var(rd, src, REG_ITMP1);
 487                                         x86_64_pop_reg(cd, d);
 488                                         store_reg_to_var_int(src, d);
 489
 490                                 } else if (bptr->type == BBTYPE_EXH) {
 491                                         d = reg_of_var(rd, src, REG_ITMP1);
 492                                         M_INTMOVE(REG_ITMP1, d);
 493                                         store_reg_to_var_int(src, d);
 494                                 }
 495
 496                         } else {
 497                                 d = reg_of_var(rd, src, REG_ITMP1);
 498                                 if ((src->varkind != STACKVAR)) {
 499                                         s2 = src->type;
 500                                         if (IS_FLT_DBL_TYPE(s2)) {
 501                                                 s1 = rd->interfaces[len][s2].regoff;
 502                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 503                                                         M_FLTMOVE(s1, d);
 504
 505                                                 } else {
 506                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 507                                                 }
 508                                                 store_reg_to_var_flt(src, d);
 509
 510                                         } else {
 511                                                 s1 = rd->interfaces[len][s2].regoff;
 512                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 513                                                         M_INTMOVE(s1, d);
 514
 515                                                 } else {
 516                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 517                                                 }
 518                                                 store_reg_to_var_int(src, d);
 519                                         }
 520                                 }
 521                         }
 522                         src = src->prev;
 523                 }
 524
 525                 /* walk through all instructions */
 526
 527                 src = bptr->instack;
 528                 len = bptr->icount;
 529                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 530
 531                         MCODECHECK(64);   /* an instruction usually needs < 64 words      */
 532                         switch (iptr->opc) {
 533
 534                         case ICMD_NOP:    /* ...  ==> ...                                 */
 535                                 break;
 536
 537                         case ICMD_NULLCHECKPOP: /* ..., objectref  ==> ...                */
 538                                 if (src->flags & INMEMORY) {
 539                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 540
 541                                 } else {
 542                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 543                                 }
 544                                 x86_64_jcc(cd, X86_64_CC_E, 0);
 545                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 546                                 break;
 547
 548                 /* constant operations ************************************************/
 549
 550                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 551                                       /* op1 = 0, val.i = constant                    */
 552
 553                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 554                         if (iptr->val.i == 0) {
 555                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 556                         } else {
 557                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 558                         }
 559                         store_reg_to_var_int(iptr->dst, d);
 560                         break;
 561
 562                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 563                                       /* op1 = 0, val.a = constant                    */
 564
 565                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 566                         if (iptr->val.a == 0) {
 567                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 568                         } else {
 569                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 570                         }
 571                         store_reg_to_var_int(iptr->dst, d);
 572                         break;
 573
 574                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 575                                       /* op1 = 0, val.l = constant                    */
 576
 577                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 578                         if (iptr->val.l == 0) {
 579                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 580                         } else {
 581                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 582                         }
 583                         store_reg_to_var_int(iptr->dst, d);
 584                         break;
 585
 586                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 587                                       /* op1 = 0, val.f = constant                    */
 588
 589                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 590                         a = dseg_addfloat(cd, iptr->val.f);
 591                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 592                         store_reg_to_var_flt(iptr->dst, d);
 593                         break;
 594
 595                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 596                                       /* op1 = 0, val.d = constant                    */
 597
 598                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 599                         a = dseg_adddouble(cd, iptr->val.d);
 600                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 601                         store_reg_to_var_flt(iptr->dst, d);
 602                         break;
 603
 604
 605                 /* load/store operations **********************************************/
 606
 607                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 608                                       /* op1 = local variable                         */
 609
 610                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 611                         if ((iptr->dst->varkind == LOCALVAR) &&
 612                             (iptr->dst->varnum == iptr->op1)) {
 613                                 break;
 614                         }
 615                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 616                         if (var->flags & INMEMORY) {
 617                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 618                                 store_reg_to_var_int(iptr->dst, d);
 619
 620                         } else {
 621                                 if (iptr->dst->flags & INMEMORY) {
 622                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 623
 624                                 } else {
 625                                         M_INTMOVE(var->regoff, d);
 626                                 }
 627                         }
 628                         break;
 629
 630                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 631                 case ICMD_ALOAD:      /* op1 = local variable                         */
 632
 633                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 634                         if ((iptr->dst->varkind == LOCALVAR) &&
 635                             (iptr->dst->varnum == iptr->op1)) {
 636                                 break;
 637                         }
 638                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 639                         if (var->flags & INMEMORY) {
 640                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 641                                 store_reg_to_var_int(iptr->dst, d);
 642
 643                         } else {
 644                                 if (iptr->dst->flags & INMEMORY) {
 645                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 646
 647                                 } else {
 648                                         M_INTMOVE(var->regoff, d);
 649                                 }
 650                         }
 651                         break;
 652
 653                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 654                 case ICMD_DLOAD:      /* op1 = local variable                         */
 655
 656                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 657                         if ((iptr->dst->varkind == LOCALVAR) &&
 658                             (iptr->dst->varnum == iptr->op1)) {
 659                                 break;
 660                         }
 661                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 662                         if (var->flags & INMEMORY) {
 663                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 664                                 store_reg_to_var_flt(iptr->dst, d);
 665
 666                         } else {
 667                                 if (iptr->dst->flags & INMEMORY) {
 668                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 669
 670                                 } else {
 671                                         M_FLTMOVE(var->regoff, d);
 672                                 }
 673                         }
 674                         break;
 675
 676                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 677                 case ICMD_LSTORE:     /* op1 = local variable                         */
 678                 case ICMD_ASTORE:
 679
 680                         if ((src->varkind == LOCALVAR) &&
 681                             (src->varnum == iptr->op1)) {
 682                                 break;
 683                         }
 684                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 685                         if (var->flags & INMEMORY) {
 686                                 var_to_reg_int(s1, src, REG_ITMP1);
 687                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 688
 689                         } else {
 690                                 var_to_reg_int(s1, src, var->regoff);
 691                                 M_INTMOVE(s1, var->regoff);
 692                         }
 693                         break;
 694
 695                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 696                 case ICMD_DSTORE:     /* op1 = local variable                         */
 697
 698                         if ((src->varkind == LOCALVAR) &&
 699                             (src->varnum == iptr->op1)) {
 700                                 break;
 701                         }
 702                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 703                         if (var->flags & INMEMORY) {
 704                                 var_to_reg_flt(s1, src, REG_FTMP1);
 705                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 706
 707                         } else {
 708                                 var_to_reg_flt(s1, src, var->regoff);
 709                                 M_FLTMOVE(s1, var->regoff);
 710                         }
 711                         break;
 712
 713
 714                 /* pop/dup/swap operations ********************************************/
 715
 716                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 717
 718                 case ICMD_POP:        /* ..., value  ==> ...                          */
 719                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 720                         break;
 721
 722                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 723                         M_COPY(src, iptr->dst);
 724                         break;
 725
 726                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 727
 728                         M_COPY(src,       iptr->dst);
 729                         M_COPY(src->prev, iptr->dst->prev);
 730                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 731                         break;
 732
 733                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 734
 735                         M_COPY(src,             iptr->dst);
 736                         M_COPY(src->prev,       iptr->dst->prev);
 737                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 738                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 739                         break;
 740
 741                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 742
 743                         M_COPY(src,       iptr->dst);
 744                         M_COPY(src->prev, iptr->dst->prev);
 745                         break;
 746
 747                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 748
 749                         M_COPY(src,             iptr->dst);
 750                         M_COPY(src->prev,       iptr->dst->prev);
 751                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 752                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 753                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 754                         break;
 755
 756                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 757
 758                         M_COPY(src,                   iptr->dst);
 759                         M_COPY(src->prev,             iptr->dst->prev);
 760                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 761                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 762                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 763                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 764                         break;
 765
 766                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 767
 768                         M_COPY(src,       iptr->dst->prev);
 769                         M_COPY(src->prev, iptr->dst);
 770                         break;
 771
 772
 773                 /* integer operations *************************************************/
 774
 775                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 776
 777                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 778                         if (iptr->dst->flags & INMEMORY) {
 779                                 if (src->flags & INMEMORY) {
 780                                         if (src->regoff == iptr->dst->regoff) {
 781                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 782
 783                                         } else {
 784                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 785                                                 x86_64_negl_reg(cd, REG_ITMP1);
 786                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 787                                         }
 788
 789                                 } else {
 790                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 791                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 792                                 }
 793
 794                         } else {
 795                                 if (src->flags & INMEMORY) {
 796                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 797                                         x86_64_negl_reg(cd, d);
 798
 799                                 } else {
 800                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 801                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 802                                 }
 803                         }
 804                         break;
 805
 806                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 807
 808                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 809                         if (iptr->dst->flags & INMEMORY) {
 810                                 if (src->flags & INMEMORY) {
 811                                         if (src->regoff == iptr->dst->regoff) {
 812                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 813
 814                                         } else {
 815                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 816                                                 x86_64_neg_reg(cd, REG_ITMP1);
 817                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 818                                         }
 819
 820                                 } else {
 821                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 822                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 823                                 }
 824
 825                         } else {
 826                                 if (src->flags & INMEMORY) {
 827                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 828                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 829
 830                                 } else {
 831                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 832                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 833                                 }
 834                         }
 835                         break;
 836
 837                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 838
 839                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 840                         if (src->flags & INMEMORY) {
 841                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 842
 843                         } else {
 844                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 845                         }
 846                         store_reg_to_var_int(iptr->dst, d);
 847                         break;
 848
 849                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 850
 851                         var_to_reg_int(s1, src, REG_ITMP1);
 852                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 853                         M_INTMOVE(s1, d);
 854                         store_reg_to_var_int(iptr->dst, d);
 855                         break;
 856
 857                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 858
 859                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 860                         if (src->flags & INMEMORY) {
 861                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 862
 863                         } else {
 864                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 865                         }
 866                         store_reg_to_var_int(iptr->dst, d);
 867                         break;
 868
 869                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 870
 871                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 872                         if (src->flags & INMEMORY) {
 873                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 874
 875                         } else {
 876                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 877                         }
 878                         store_reg_to_var_int(iptr->dst, d);
 879                         break;
 880
 881                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 882
 883                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 884                         if (src->flags & INMEMORY) {
 885                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 886
 887                         } else {
 888                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 889                         }
 890                         store_reg_to_var_int(iptr->dst, d);
 891                         break;
 892
 893
 894                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 895
 896                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 897                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 898                         break;
 899
 900                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 901                                       /* val.i = constant                             */
 902
 903                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 904                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 905                         break;
 906
 907                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 908
 909                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 910                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 911                         break;
 912
 913                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 914                                       /* val.l = constant                             */
 915
 916                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 917                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 918                         break;
 919
 920                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 921
 922                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 923                         if (iptr->dst->flags & INMEMORY) {
 924                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 925                                         if (src->prev->regoff == iptr->dst->regoff) {
 926                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 927                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 928
 929                                         } else {
 930                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 931                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 932                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 933                                         }
 934
 935                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 936                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 937                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 938                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 939
 940                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 941                                         if (src->prev->regoff == iptr->dst->regoff) {
 942                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 943
 944                                         } else {
 945                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 946                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 947                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 948                                         }
 949
 950                                 } else {
 951                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 952                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 953                                 }
 954
 955                         } else {
 956                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 957                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 958                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 959
 960                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 961                                         M_INTMOVE(src->prev->regoff, d);
 962                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 963
 964                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 965                                         /* workaround for reg alloc */
 966                                         if (src->regoff == iptr->dst->regoff) {
 967                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 968                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 969                                                 M_INTMOVE(REG_ITMP1, d);
 970
 971                                         } else {
 972                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 973                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 974                                         }
 975
 976                                 } else {
 977                                         /* workaround for reg alloc */
 978                                         if (src->regoff == iptr->dst->regoff) {
 979                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 980                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 981                                                 M_INTMOVE(REG_ITMP1, d);
 982
 983                                         } else {
 984                                                 M_INTMOVE(src->prev->regoff, d);
 985                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 986                                         }
 987                                 }
 988                         }
 989                         break;
 990
 991                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 992                                       /* val.i = constant                             */
 993
 994                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 995                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 996                         break;
 997
 998                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 999
1000                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1001                         if (iptr->dst->flags & INMEMORY) {
1002                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1003                                         if (src->prev->regoff == iptr->dst->regoff) {
1004                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1005                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1006
1007                                         } else {
1008                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1009                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1010                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1011                                         }
1012
1013                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1014                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1015                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1016                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1017
1018                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1019                                         if (src->prev->regoff == iptr->dst->regoff) {
1020                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1021
1022                                         } else {
1023                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1024                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1025                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1026                                         }
1027
1028                                 } else {
1029                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1030                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1031                                 }
1032
1033                         } else {
1034                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1035                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1036                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1037
1038                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1039                                         M_INTMOVE(src->prev->regoff, d);
1040                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1041
1042                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1043                                         /* workaround for reg alloc */
1044                                         if (src->regoff == iptr->dst->regoff) {
1045                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1046                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1047                                                 M_INTMOVE(REG_ITMP1, d);
1048
1049                                         } else {
1050                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1051                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1052                                         }
1053
1054                                 } else {
1055                                         /* workaround for reg alloc */
1056                                         if (src->regoff == iptr->dst->regoff) {
1057                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1058                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1059                                                 M_INTMOVE(REG_ITMP1, d);
1060
1061                                         } else {
1062                                                 M_INTMOVE(src->prev->regoff, d);
1063                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1064                                         }
1065                                 }
1066                         }
1067                         break;
1068
1069                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1070                                       /* val.l = constant                             */
1071
1072                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1073                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1074                         break;
1075
1076                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1077
1078                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1079                         if (iptr->dst->flags & INMEMORY) {
1080                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1081                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1082                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1083                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1084
1085                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1086                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1087                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1088                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1089
1090                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1091                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1092                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1093                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1094
1095                                 } else {
1096                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1097                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1098                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1099                                 }
1100
1101                         } else {
1102                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1103                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1104                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1105
1106                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1107                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1108                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1109
1110                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1111                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1112                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1113
1114                                 } else {
1115                                         if (src->regoff == iptr->dst->regoff) {
1116                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1117
1118                                         } else {
1119                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1120                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1121                                         }
1122                                 }
1123                         }
1124                         break;
1125
1126                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1127                                       /* val.i = constant                             */
1128
1129                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1130                         if (iptr->dst->flags & INMEMORY) {
1131                                 if (src->flags & INMEMORY) {
1132                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1133                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1134
1135                                 } else {
1136                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1137                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1138                                 }
1139
1140                         } else {
1141                                 if (src->flags & INMEMORY) {
1142                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1143
1144                                 } else {
1145                                         if (iptr->val.i == 2) {
1146                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1147                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1148
1149                                         } else {
1150                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1151                                         }
1152                                 }
1153                         }
1154                         break;
1155
1156                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1157
1158                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1159                         if (iptr->dst->flags & INMEMORY) {
1160                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1161                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1162                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1163                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1164
1165                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1166                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1167                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1168                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1169
1170                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1171                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1172                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1173                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1174
1175                                 } else {
1176                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1177                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1178                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1179                                 }
1180
1181                         } else {
1182                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1183                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1184                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1185
1186                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1187                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1188                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1189
1190                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1191                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1192                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1193
1194                                 } else {
1195                                         if (src->regoff == iptr->dst->regoff) {
1196                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1197
1198                                         } else {
1199                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1200                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1201                                         }
1202                                 }
1203                         }
1204                         break;
1205
1206                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1207                                       /* val.l = constant                             */
1208
1209                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1210                         if (iptr->dst->flags & INMEMORY) {
1211                                 if (src->flags & INMEMORY) {
1212                                         if (x86_64_is_imm32(iptr->val.l)) {
1213                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1214
1215                                         } else {
1216                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1217                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1218                                         }
1219                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1220
1221                                 } else {
1222                                         if (x86_64_is_imm32(iptr->val.l)) {
1223                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1224
1225                                         } else {
1226                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1227                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1228                                         }
1229                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1230                                 }
1231
1232                         } else {
1233                                 if (src->flags & INMEMORY) {
1234                                         if (x86_64_is_imm32(iptr->val.l)) {
1235                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1236
1237                                         } else {
1238                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1239                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1240                                         }
1241
1242                                 } else {
1243                                         /* should match in many cases */
1244                                         if (iptr->val.l == 2) {
1245                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1246                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1247
1248                                         } else {
1249                                                 if (x86_64_is_imm32(iptr->val.l)) {
1250                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1251
1252                                                 } else {
1253                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1254                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1255                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1256                                                 }
1257                                         }
1258                                 }
1259                         }
1260                         break;
1261
1262                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1263
1264                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1265                 if (src->prev->flags & INMEMORY) {
1266                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1267
1268                         } else {
1269                                 M_INTMOVE(src->prev->regoff, RAX);
1270                         }
1271
1272                         if (src->flags & INMEMORY) {
1273                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1274
1275                         } else {
1276                                 M_INTMOVE(src->regoff, REG_ITMP3);
1277                         }
1278                         gen_div_check(src);
1279
1280                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1281                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1282                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1283                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1284
1285                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1286                         x86_64_cltd(cd);
1287                         x86_64_idivl_reg(cd, REG_ITMP3);
1288
1289                         if (iptr->dst->flags & INMEMORY) {
1290                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1291                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1292
1293                         } else {
1294                                 M_INTMOVE(RAX, iptr->dst->regoff);
1295
1296                                 if (iptr->dst->regoff != RDX) {
1297                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1298                                 }
1299                         }
1300                         break;
1301
1302                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1303
1304                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1305                         if (src->prev->flags & INMEMORY) {
1306                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1307
1308                         } else {
1309                                 M_INTMOVE(src->prev->regoff, RAX);
1310                         }
1311
1312                         if (src->flags & INMEMORY) {
1313                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1314
1315                         } else {
1316                                 M_INTMOVE(src->regoff, REG_ITMP3);
1317                         }
1318                         gen_div_check(src);
1319
1320                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1321                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1322                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1323                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1324                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1325
1326                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1327                         x86_64_cltd(cd);
1328                         x86_64_idivl_reg(cd, REG_ITMP3);
1329
1330                         if (iptr->dst->flags & INMEMORY) {
1331                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1332                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1333
1334                         } else {
1335                                 M_INTMOVE(RDX, iptr->dst->regoff);
1336
1337                                 if (iptr->dst->regoff != RDX) {
1338                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1339                                 }
1340                         }
1341                         break;
1342
1343                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1344                                       /* val.i = constant                             */
1345
1346                         var_to_reg_int(s1, src, REG_ITMP1);
1347                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1348                         M_INTMOVE(s1, REG_ITMP1);
1349                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1350                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1351                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1352                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1353                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1354                         store_reg_to_var_int(iptr->dst, d);
1355                         break;
1356
1357                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1358                                       /* val.i = constant                             */
1359
1360                         var_to_reg_int(s1, src, REG_ITMP1);
1361                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1362                         M_INTMOVE(s1, REG_ITMP1);
1363                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1364                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1365                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1366                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1367                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1368                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1369                         store_reg_to_var_int(iptr->dst, d);
1370                         break;
1371
1372
1373                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1374
1375                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1376                 if (src->prev->flags & INMEMORY) {
1377                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1378
1379                         } else {
1380                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1381                         }
1382
1383                         if (src->flags & INMEMORY) {
1384                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1385
1386                         } else {
1387                                 M_INTMOVE(src->regoff, REG_ITMP3);
1388                         }
1389                         gen_div_check(src);
1390
1391                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1392                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1393                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1394                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1395                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1396
1397                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1398                         x86_64_cqto(cd);
1399                         x86_64_idiv_reg(cd, REG_ITMP3);
1400
1401                         if (iptr->dst->flags & INMEMORY) {
1402                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1403                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1404
1405                         } else {
1406                                 M_INTMOVE(RAX, iptr->dst->regoff);
1407
1408                                 if (iptr->dst->regoff != RDX) {
1409                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1410                                 }
1411                         }
1412                         break;
1413
1414                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1415
1416                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1417                         if (src->prev->flags & INMEMORY) {
1418                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1419
1420                         } else {
1421                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1422                         }
1423
1424                         if (src->flags & INMEMORY) {
1425                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1426
1427                         } else {
1428                                 M_INTMOVE(src->regoff, REG_ITMP3);
1429                         }
1430                         gen_div_check(src);
1431
1432                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1433                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1434                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1435                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1436                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1437                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1438
1439                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1440                         x86_64_cqto(cd);
1441                         x86_64_idiv_reg(cd, REG_ITMP3);
1442
1443                         if (iptr->dst->flags & INMEMORY) {
1444                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1445                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1446
1447                         } else {
1448                                 M_INTMOVE(RDX, iptr->dst->regoff);
1449
1450                                 if (iptr->dst->regoff != RDX) {
1451                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1452                                 }
1453                         }
1454                         break;
1455
1456                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1457                                       /* val.i = constant                             */
1458
1459                         var_to_reg_int(s1, src, REG_ITMP1);
1460                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1461                         M_INTMOVE(s1, REG_ITMP1);
1462                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1463                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1464                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1465                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1466                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1467                         store_reg_to_var_int(iptr->dst, d);
1468                         break;
1469
1470                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1471                                       /* val.l = constant                             */
1472
1473                         var_to_reg_int(s1, src, REG_ITMP1);
1474                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1475                         M_INTMOVE(s1, REG_ITMP1);
1476                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1477                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1478                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1479                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1480                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1481                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1482                         store_reg_to_var_int(iptr->dst, d);
1483                         break;
1484
1485                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1486
1487                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1488                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1489                         break;
1490
1491                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1492                                       /* val.i = constant                             */
1493
1494                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1495                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1496                         break;
1497
1498                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1499
1500                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1501                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1502                         break;
1503
1504                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1505                                       /* val.i = constant                             */
1506
1507                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1508                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1509                         break;
1510
1511                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1512
1513                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1514                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1515                         break;
1516
1517                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1518                                       /* val.i = constant                             */
1519
1520                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1521                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1522                         break;
1523
1524                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1525
1526                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1527                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1528                         break;
1529
1530         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1531                                           /* val.i = constant                             */
1532
1533                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1534                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1535                         break;
1536
1537                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1538
1539                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1540                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1541                         break;
1542
1543                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1544                                       /* val.i = constant                             */
1545
1546                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1547                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1548                         break;
1549
1550                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1551
1552                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1553                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1554                         break;
1555
1556                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1557                                       /* val.l = constant                             */
1558
1559                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1560                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1561                         break;
1562
1563                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1564
1565                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1566                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1567                         break;
1568
1569                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1570                                       /* val.i = constant                             */
1571
1572                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1573                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1574                         break;
1575
1576                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1577
1578                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1579                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1580                         break;
1581
1582                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1583                                       /* val.l = constant                             */
1584
1585                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1586                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1587                         break;
1588
1589                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1590
1591                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1592                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1593                         break;
1594
1595                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1596                                       /* val.i = constant                             */
1597
1598                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1599                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1600                         break;
1601
1602                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1603
1604                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1605                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1606                         break;
1607
1608                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1609                                       /* val.l = constant                             */
1610
1611                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1612                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1613                         break;
1614
1615                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1616
1617                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1618                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1619                         break;
1620
1621                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1622                                       /* val.i = constant                             */
1623
1624                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1625                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1626                         break;
1627
1628                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1629
1630                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1631                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1632                         break;
1633
1634                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1635                                       /* val.l = constant                             */
1636
1637                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1638                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1639                         break;
1640
1641
1642                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1643                                       /* op1 = variable, val.i = constant             */
1644
1645                         /* using inc and dec is definitely faster than add -- tested      */
1646                         /* with sieve                                                     */
1647
1648                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1649                         d = var->regoff;
1650                         if (var->flags & INMEMORY) {
1651                                 if (iptr->val.i == 1) {
1652                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1653
1654                                 } else if (iptr->val.i == -1) {
1655                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1656
1657                                 } else {
1658                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1659                                 }
1660
1661                         } else {
1662                                 if (iptr->val.i == 1) {
1663                                         x86_64_incl_reg(cd, d);
1664
1665                                 } else if (iptr->val.i == -1) {
1666                                         x86_64_decl_reg(cd, d);
1667
1668                                 } else {
1669                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1670                                 }
1671                         }
1672                         break;
1673
1674
1675                 /* floating operations ************************************************/
1676
1677                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1678
1679                         var_to_reg_flt(s1, src, REG_FTMP1);
1680                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1681                         a = dseg_adds4(cd, 0x80000000);
1682                         M_FLTMOVE(s1, d);
1683                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1684                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1685                         store_reg_to_var_flt(iptr->dst, d);
1686                         break;
1687
1688                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1689
1690                         var_to_reg_flt(s1, src, REG_FTMP1);
1691                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1692                         a = dseg_adds8(cd, 0x8000000000000000);
1693                         M_FLTMOVE(s1, d);
1694                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1695                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1696                         store_reg_to_var_flt(iptr->dst, d);
1697                         break;
1698
1699                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1700
1701                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1702                         var_to_reg_flt(s2, src, REG_FTMP2);
1703                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1704                         if (s1 == d) {
1705                                 x86_64_addss_reg_reg(cd, s2, d);
1706                         } else if (s2 == d) {
1707                                 x86_64_addss_reg_reg(cd, s1, d);
1708                         } else {
1709                                 M_FLTMOVE(s1, d);
1710                                 x86_64_addss_reg_reg(cd, s2, d);
1711                         }
1712                         store_reg_to_var_flt(iptr->dst, d);
1713                         break;
1714
1715                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1716
1717                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1718                         var_to_reg_flt(s2, src, REG_FTMP2);
1719                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1720                         if (s1 == d) {
1721                                 x86_64_addsd_reg_reg(cd, s2, d);
1722                         } else if (s2 == d) {
1723                                 x86_64_addsd_reg_reg(cd, s1, d);
1724                         } else {
1725                                 M_FLTMOVE(s1, d);
1726                                 x86_64_addsd_reg_reg(cd, s2, d);
1727                         }
1728                         store_reg_to_var_flt(iptr->dst, d);
1729                         break;
1730
1731                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1732
1733                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1734                         var_to_reg_flt(s2, src, REG_FTMP2);
1735                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1736                         if (s2 == d) {
1737                                 M_FLTMOVE(s2, REG_FTMP2);
1738                                 s2 = REG_FTMP2;
1739                         }
1740                         M_FLTMOVE(s1, d);
1741                         x86_64_subss_reg_reg(cd, s2, d);
1742                         store_reg_to_var_flt(iptr->dst, d);
1743                         break;
1744
1745                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1746
1747                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1748                         var_to_reg_flt(s2, src, REG_FTMP2);
1749                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1750                         if (s2 == d) {
1751                                 M_FLTMOVE(s2, REG_FTMP2);
1752                                 s2 = REG_FTMP2;
1753                         }
1754                         M_FLTMOVE(s1, d);
1755                         x86_64_subsd_reg_reg(cd, s2, d);
1756                         store_reg_to_var_flt(iptr->dst, d);
1757                         break;
1758
1759                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1760
1761                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1762                         var_to_reg_flt(s2, src, REG_FTMP2);
1763                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1764                         if (s1 == d) {
1765                                 x86_64_mulss_reg_reg(cd, s2, d);
1766                         } else if (s2 == d) {
1767                                 x86_64_mulss_reg_reg(cd, s1, d);
1768                         } else {
1769                                 M_FLTMOVE(s1, d);
1770                                 x86_64_mulss_reg_reg(cd, s2, d);
1771                         }
1772                         store_reg_to_var_flt(iptr->dst, d);
1773                         break;
1774
1775                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1776
1777                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1778                         var_to_reg_flt(s2, src, REG_FTMP2);
1779                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1780                         if (s1 == d) {
1781                                 x86_64_mulsd_reg_reg(cd, s2, d);
1782                         } else if (s2 == d) {
1783                                 x86_64_mulsd_reg_reg(cd, s1, d);
1784                         } else {
1785                                 M_FLTMOVE(s1, d);
1786                                 x86_64_mulsd_reg_reg(cd, s2, d);
1787                         }
1788                         store_reg_to_var_flt(iptr->dst, d);
1789                         break;
1790
1791                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1792
1793                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1794                         var_to_reg_flt(s2, src, REG_FTMP2);
1795                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1796                         if (s2 == d) {
1797                                 M_FLTMOVE(s2, REG_FTMP2);
1798                                 s2 = REG_FTMP2;
1799                         }
1800                         M_FLTMOVE(s1, d);
1801                         x86_64_divss_reg_reg(cd, s2, d);
1802                         store_reg_to_var_flt(iptr->dst, d);
1803                         break;
1804
1805                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1806
1807                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1808                         var_to_reg_flt(s2, src, REG_FTMP2);
1809                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1810                         if (s2 == d) {
1811                                 M_FLTMOVE(s2, REG_FTMP2);
1812                                 s2 = REG_FTMP2;
1813                         }
1814                         M_FLTMOVE(s1, d);
1815                         x86_64_divsd_reg_reg(cd, s2, d);
1816                         store_reg_to_var_flt(iptr->dst, d);
1817                         break;
1818
1819                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1820
1821                         var_to_reg_int(s1, src, REG_ITMP1);
1822                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1823                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1824                         store_reg_to_var_flt(iptr->dst, d);
1825                         break;
1826
1827                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1828
1829                         var_to_reg_int(s1, src, REG_ITMP1);
1830                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1831                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1832                         store_reg_to_var_flt(iptr->dst, d);
1833                         break;
1834
1835                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1836
1837                         var_to_reg_int(s1, src, REG_ITMP1);
1838                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1839                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1840                         store_reg_to_var_flt(iptr->dst, d);
1841                         break;
1842
1843                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1844
1845                         var_to_reg_int(s1, src, REG_ITMP1);
1846                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1847                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1848                         store_reg_to_var_flt(iptr->dst, d);
1849                         break;
1850
1851                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1852
1853                         var_to_reg_flt(s1, src, REG_FTMP1);
1854                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1855                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1856                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1857                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1858                         x86_64_jcc(cd, X86_64_CC_NE, a);
1859                         M_FLTMOVE(s1, REG_FTMP1);
1860                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1861                         x86_64_call_reg(cd, REG_ITMP2);
1862                         M_INTMOVE(REG_RESULT, d);
1863                         store_reg_to_var_int(iptr->dst, d);
1864                         break;
1865
1866                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1867
1868                         var_to_reg_flt(s1, src, REG_FTMP1);
1869                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1870                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1871                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1872                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1873                         x86_64_jcc(cd, X86_64_CC_NE, a);
1874                         M_FLTMOVE(s1, REG_FTMP1);
1875                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1876                         x86_64_call_reg(cd, REG_ITMP2);
1877                         M_INTMOVE(REG_RESULT, d);
1878                         store_reg_to_var_int(iptr->dst, d);
1879                         break;
1880
1881                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1882
1883                         var_to_reg_flt(s1, src, REG_FTMP1);
1884                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1885                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1886                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1887                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1888                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1889                         x86_64_jcc(cd, X86_64_CC_NE, a);
1890                         M_FLTMOVE(s1, REG_FTMP1);
1891                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1892                         x86_64_call_reg(cd, REG_ITMP2);
1893                         M_INTMOVE(REG_RESULT, d);
1894                         store_reg_to_var_int(iptr->dst, d);
1895                         break;
1896
1897                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1898
1899                         var_to_reg_flt(s1, src, REG_FTMP1);
1900                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1901                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1902                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1903                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1904                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1905                         x86_64_jcc(cd, X86_64_CC_NE, a);
1906                         M_FLTMOVE(s1, REG_FTMP1);
1907                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1908                         x86_64_call_reg(cd, REG_ITMP2);
1909                         M_INTMOVE(REG_RESULT, d);
1910                         store_reg_to_var_int(iptr->dst, d);
1911                         break;
1912
1913                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1914
1915                         var_to_reg_flt(s1, src, REG_FTMP1);
1916                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1917                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1918                         store_reg_to_var_flt(iptr->dst, d);
1919                         break;
1920
1921                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1922
1923                         var_to_reg_flt(s1, src, REG_FTMP1);
1924                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1925                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1926                         store_reg_to_var_flt(iptr->dst, d);
1927                         break;
1928
1929                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1930                                           /* == => 0, < => 1, > => -1 */
1931
1932                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1933                         var_to_reg_flt(s2, src, REG_FTMP2);
1934                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1935                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1936                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1937                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1938                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1939                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1940                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1941                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1942                         store_reg_to_var_int(iptr->dst, d);
1943                         break;
1944
1945                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1946                                           /* == => 0, < => 1, > => -1 */
1947
1948                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1949                         var_to_reg_flt(s2, src, REG_FTMP2);
1950                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1951                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1952                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1953                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1954                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1955                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1956                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1957                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1958                         store_reg_to_var_int(iptr->dst, d);
1959                         break;
1960
1961                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1962                                           /* == => 0, < => 1, > => -1 */
1963
1964                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1965                         var_to_reg_flt(s2, src, REG_FTMP2);
1966                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1967                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1968                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1969                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1970                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1971                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1972                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1973                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1974                         store_reg_to_var_int(iptr->dst, d);
1975                         break;
1976
1977                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1978                                           /* == => 0, < => 1, > => -1 */
1979
1980                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1981                         var_to_reg_flt(s2, src, REG_FTMP2);
1982                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1983                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1984                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1985                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1986                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1987                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1988                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1989                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1990                         store_reg_to_var_int(iptr->dst, d);
1991                         break;
1992
1993
1994                 /* memory operations **************************************************/
1995
1996                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1997
1998                         var_to_reg_int(s1, src, REG_ITMP1);
1999                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2000                         gen_nullptr_check(s1);
2001                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
2002                         store_reg_to_var_int(iptr->dst, d);
2003                         break;
2004
2005                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2006
2007                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2008                         var_to_reg_int(s2, src, REG_ITMP2);
2009                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2010                         if (iptr->op1 == 0) {
2011                                 gen_nullptr_check(s1);
2012                                 gen_bound_check;
2013                         }
2014                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2015                         store_reg_to_var_int(iptr->dst, d);
2016                         break;
2017
2018                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2019
2020                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2021                         var_to_reg_int(s2, src, REG_ITMP2);
2022                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2023                         if (iptr->op1 == 0) {
2024                                 gen_nullptr_check(s1);
2025                                 gen_bound_check;
2026                         }
2027                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2028                         store_reg_to_var_int(iptr->dst, d);
2029                         break;
2030
2031                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2032
2033                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2034                         var_to_reg_int(s2, src, REG_ITMP2);
2035                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2036                         if (iptr->op1 == 0) {
2037                                 gen_nullptr_check(s1);
2038                                 gen_bound_check;
2039                         }
2040                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2041                         store_reg_to_var_int(iptr->dst, d);
2042                         break;
2043
2044                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2045
2046                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2047                         var_to_reg_int(s2, src, REG_ITMP2);
2048                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2049                         if (iptr->op1 == 0) {
2050                                 gen_nullptr_check(s1);
2051                                 gen_bound_check;
2052                         }
2053                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2054                         store_reg_to_var_flt(iptr->dst, d);
2055                         break;
2056
2057                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2058
2059                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2060                         var_to_reg_int(s2, src, REG_ITMP2);
2061                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2062                         if (iptr->op1 == 0) {
2063                                 gen_nullptr_check(s1);
2064                                 gen_bound_check;
2065                         }
2066                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2067                         store_reg_to_var_flt(iptr->dst, d);
2068                         break;
2069
2070                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2071
2072                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2073                         var_to_reg_int(s2, src, REG_ITMP2);
2074                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2075                         if (iptr->op1 == 0) {
2076                                 gen_nullptr_check(s1);
2077                                 gen_bound_check;
2078                         }
2079                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2080                         store_reg_to_var_int(iptr->dst, d);
2081                         break;
2082
2083                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2084
2085                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2086                         var_to_reg_int(s2, src, REG_ITMP2);
2087                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2088                         if (iptr->op1 == 0) {
2089                                 gen_nullptr_check(s1);
2090                                 gen_bound_check;
2091                         }
2092                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2093                         store_reg_to_var_int(iptr->dst, d);
2094                         break;
2095
2096                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2097
2098                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2099                         var_to_reg_int(s2, src, REG_ITMP2);
2100                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2101                         if (iptr->op1 == 0) {
2102                                 gen_nullptr_check(s1);
2103                                 gen_bound_check;
2104                         }
2105                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2106                         store_reg_to_var_int(iptr->dst, d);
2107                         break;
2108
2109
2110                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2111
2112                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2113                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2114                         if (iptr->op1 == 0) {
2115                                 gen_nullptr_check(s1);
2116                                 gen_bound_check;
2117                         }
2118                         var_to_reg_int(s3, src, REG_ITMP3);
2119                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2120                         break;
2121
2122                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2123
2124                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2125                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2126                         if (iptr->op1 == 0) {
2127                                 gen_nullptr_check(s1);
2128                                 gen_bound_check;
2129                         }
2130                         var_to_reg_int(s3, src, REG_ITMP3);
2131                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2132                         break;
2133
2134                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2135
2136                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2137                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2138                         if (iptr->op1 == 0) {
2139                                 gen_nullptr_check(s1);
2140                                 gen_bound_check;
2141                         }
2142                         var_to_reg_int(s3, src, REG_ITMP3);
2143                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2144                         break;
2145
2146                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2147
2148                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2149                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2150                         if (iptr->op1 == 0) {
2151                                 gen_nullptr_check(s1);
2152                                 gen_bound_check;
2153                         }
2154                         var_to_reg_flt(s3, src, REG_FTMP3);
2155                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2156                         break;
2157
2158                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2159
2160                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2161                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2162                         if (iptr->op1 == 0) {
2163                                 gen_nullptr_check(s1);
2164                                 gen_bound_check;
2165                         }
2166                         var_to_reg_flt(s3, src, REG_FTMP3);
2167                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2168                         break;
2169
2170                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2171
2172                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2173                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2174                         if (iptr->op1 == 0) {
2175                                 gen_nullptr_check(s1);
2176                                 gen_bound_check;
2177                         }
2178                         var_to_reg_int(s3, src, REG_ITMP3);
2179                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2180                         break;
2181
2182                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2183
2184                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2185                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2186                         if (iptr->op1 == 0) {
2187                                 gen_nullptr_check(s1);
2188                                 gen_bound_check;
2189                         }
2190                         var_to_reg_int(s3, src, REG_ITMP3);
2191                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2192                         break;
2193
2194                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2195
2196                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2197                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2198                         if (iptr->op1 == 0) {
2199                                 gen_nullptr_check(s1);
2200                                 gen_bound_check;
2201                         }
2202                         var_to_reg_int(s3, src, REG_ITMP3);
2203                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2204                         break;
2205
2206                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2207
2208                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2209                         var_to_reg_int(s2, src, REG_ITMP2);
2210                         if (iptr->op1 == 0) {
2211                                 gen_nullptr_check(s1);
2212                                 gen_bound_check;
2213                         }
2214                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2215                         break;
2216
2217                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2218
2219                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2220                         var_to_reg_int(s2, src, REG_ITMP2);
2221                         if (iptr->op1 == 0) {
2222                                 gen_nullptr_check(s1);
2223                                 gen_bound_check;
2224                         }
2225
2226                         if (x86_64_is_imm32(iptr->val.l)) {
2227                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2228
2229                         } else {
2230                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2231                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2232                         }
2233                         break;
2234
2235                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2236
2237                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2238                         var_to_reg_int(s2, src, REG_ITMP2);
2239                         if (iptr->op1 == 0) {
2240                                 gen_nullptr_check(s1);
2241                                 gen_bound_check;
2242                         }
2243                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2244                         break;
2245
2246                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2247
2248                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2249                         var_to_reg_int(s2, src, REG_ITMP2);
2250                         if (iptr->op1 == 0) {
2251                                 gen_nullptr_check(s1);
2252                                 gen_bound_check;
2253                         }
2254                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2255                         break;
2256
2257                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2258
2259                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2260                         var_to_reg_int(s2, src, REG_ITMP2);
2261                         if (iptr->op1 == 0) {
2262                                 gen_nullptr_check(s1);
2263                                 gen_bound_check;
2264                         }
2265                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2266                         break;
2267
2268                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2269
2270                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2271                         var_to_reg_int(s2, src, REG_ITMP2);
2272                         if (iptr->op1 == 0) {
2273                                 gen_nullptr_check(s1);
2274                                 gen_bound_check;
2275                         }
2276                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2277                         break;
2278
2279
2280                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2281                                       /* op1 = type, val.a = field address            */
2282
2283                         /* If the static fields' class is not yet initialized, we do it   */
2284                         /* now. The call code is generated later.                         */
2285                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2286                                 codegen_addclinitref(cd, cd->mcodeptr, ((fieldinfo *) iptr->val.a)->class);
2287
2288                                 /* This is just for debugging purposes. Is very difficult to  */
2289                                 /* read patched code. Here we patch the following 5 nop's     */
2290                                 /* so that the real code keeps untouched.                     */
2291                                 if (showdisassemble) {
2292                                         x86_64_nop(cd);
2293                                         x86_64_nop(cd);
2294                                         x86_64_nop(cd);
2295                                         x86_64_nop(cd);
2296                                         x86_64_nop(cd);
2297                                 }
2298                         }
2299
2300                         /* This approach is much faster than moving the field address     */
2301                         /* inline into a register. */
2302                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2303                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2304                         switch (iptr->op1) {
2305                         case TYPE_INT:
2306                                 var_to_reg_int(s2, src, REG_ITMP1);
2307                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2308                                 break;
2309                         case TYPE_LNG:
2310                         case TYPE_ADR:
2311                                 var_to_reg_int(s2, src, REG_ITMP1);
2312                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2313                                 break;
2314                         case TYPE_FLT:
2315                                 var_to_reg_flt(s2, src, REG_FTMP1);
2316                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2317                                 break;
2318                         case TYPE_DBL:
2319                                 var_to_reg_flt(s2, src, REG_FTMP1);
2320                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2321                                 break;
2322                         default:
2323                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2324                                                                                    "Unknown PUTSTATIC operand type %d",
2325                                                                                    iptr->op1);
2326                         }
2327                         break;
2328
2329                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2330                                       /* op1 = type, val.a = field address            */
2331
2332                         /* If the static fields' class is not yet initialized, we do it   */
2333                         /* now. The call code is generated later.                         */
2334                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2335                                 codegen_addclinitref(cd, cd->mcodeptr, ((fieldinfo *) iptr->val.a)->class);
2336
2337                                 /* This is just for debugging purposes. Is very difficult to  */
2338                                 /* read patched code. Here we patch the following 5 nop's     */
2339                                 /* so that the real code keeps untouched.                     */
2340                                 if (showdisassemble) {
2341                                         x86_64_nop(cd);
2342                                         x86_64_nop(cd);
2343                                         x86_64_nop(cd);
2344                                         x86_64_nop(cd);
2345                                         x86_64_nop(cd);
2346                                 }
2347                         }
2348
2349                         /* This approach is much faster than moving the field address     */
2350                         /* inline into a register. */
2351                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2352                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2353                         switch (iptr->op1) {
2354                         case TYPE_INT:
2355                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2356                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2357                                 store_reg_to_var_int(iptr->dst, d);
2358                                 break;
2359                         case TYPE_LNG:
2360                         case TYPE_ADR:
2361                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2362                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2363                                 store_reg_to_var_int(iptr->dst, d);
2364                                 break;
2365                         case TYPE_FLT:
2366                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2367                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2368                                 store_reg_to_var_flt(iptr->dst, d);
2369                                 break;
2370                         case TYPE_DBL:
2371                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2372                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2373                                 store_reg_to_var_flt(iptr->dst, d);
2374                                 break;
2375                         default:
2376                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2377                                                                                    "Unknown GETSTATIC operand type %d",
2378                                                                                    iptr->op1);
2379                         }
2380                         break;
2381
2382                 case ICMD_PUTFIELD:   /* ..., value  ==> ...                          */
2383                                       /* op1 = type, val.i = field offset             */
2384
2385                         a = ((fieldinfo *)(iptr->val.a))->offset;
2386                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2387                         switch (iptr->op1) {
2388                         case TYPE_INT:
2389                                 var_to_reg_int(s2, src, REG_ITMP2);
2390                                 gen_nullptr_check(s1);
2391                                 x86_64_movl_reg_membase(cd, s2, s1, a);
2392                                 break;
2393                         case TYPE_LNG:
2394                         case TYPE_ADR:
2395                                 var_to_reg_int(s2, src, REG_ITMP2);
2396                                 gen_nullptr_check(s1);
2397                                 x86_64_mov_reg_membase(cd, s2, s1, a);
2398                                 break;
2399                         case TYPE_FLT:
2400                                 var_to_reg_flt(s2, src, REG_FTMP2);
2401                                 gen_nullptr_check(s1);
2402                                 x86_64_movss_reg_membase(cd, s2, s1, a);
2403                                 break;
2404                         case TYPE_DBL:
2405                                 var_to_reg_flt(s2, src, REG_FTMP2);
2406                                 gen_nullptr_check(s1);
2407                                 x86_64_movsd_reg_membase(cd, s2, s1, a);
2408                                 break;
2409                         default:
2410                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2411                                                                                    "Unknown PUTFIELD operand type %d",
2412                                                                                    iptr->op1);
2413                         }
2414                         break;
2415
2416                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2417                                       /* op1 = type, val.i = field offset             */
2418
2419                         a = ((fieldinfo *)(iptr->val.a))->offset;
2420                         var_to_reg_int(s1, src, REG_ITMP1);
2421                         switch (iptr->op1) {
2422                         case TYPE_INT:
2423                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2424                                 gen_nullptr_check(s1);
2425                                 x86_64_movl_membase_reg(cd, s1, a, d);
2426                                 store_reg_to_var_int(iptr->dst, d);
2427                                 break;
2428                         case TYPE_LNG:
2429                         case TYPE_ADR:
2430                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2431                                 gen_nullptr_check(s1);
2432                                 x86_64_mov_membase_reg(cd, s1, a, d);
2433                                 store_reg_to_var_int(iptr->dst, d);
2434                                 break;
2435                         case TYPE_FLT:
2436                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2437                                 gen_nullptr_check(s1);
2438                                 x86_64_movss_membase_reg(cd, s1, a, d);
2439                                 store_reg_to_var_flt(iptr->dst, d);
2440                                 break;
2441                         case TYPE_DBL:
2442                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2443                                 gen_nullptr_check(s1);
2444                                 x86_64_movsd_membase_reg(cd, s1, a, d);
2445                                 store_reg_to_var_flt(iptr->dst, d);
2446                                 break;
2447                         default:
2448                                 throw_cacao_exception_exit(string_java_lang_InternalError,
2449                                                                                    "Unknown GETFIELD operand type %d",
2450                                                                                    iptr->op1);
2451                         }
2452                         break;
2453
2454
2455                 /* branch operations **************************************************/
2456
2457                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2458
2459                         var_to_reg_int(s1, src, REG_ITMP1);
2460                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2461
2462                         x86_64_call_imm(cd, 0); /* passing exception pointer              */
2463                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2464
2465                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
2466                         x86_64_jmp_reg(cd, REG_ITMP3);
2467                         ALIGNCODENOP;
2468                         break;
2469
2470                 case ICMD_GOTO:         /* ... ==> ...                                */
2471                                         /* op1 = target JavaVM pc                     */
2472
2473                         x86_64_jmp_imm(cd, 0);
2474                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2475                         ALIGNCODENOP;
2476                         break;
2477
2478                 case ICMD_JSR:          /* ... ==> ...                                */
2479                                         /* op1 = target JavaVM pc                     */
2480
2481                         x86_64_call_imm(cd, 0);
2482                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2483                         break;
2484
2485                 case ICMD_RET:          /* ... ==> ...                                */
2486                                         /* op1 = local variable                       */
2487
2488                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2489                         var_to_reg_int(s1, var, REG_ITMP1);
2490                         x86_64_jmp_reg(cd, s1);
2491                         break;
2492
2493                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2494                                         /* op1 = target JavaVM pc                     */
2495
2496                         if (src->flags & INMEMORY) {
2497                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2498
2499                         } else {
2500                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2501                         }
2502                         x86_64_jcc(cd, X86_64_CC_E, 0);
2503                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2504                         break;
2505
2506                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2507                                         /* op1 = target JavaVM pc                     */
2508
2509                         if (src->flags & INMEMORY) {
2510                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2511
2512                         } else {
2513                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2514                         }
2515                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2516                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2517                         break;
2518
2519                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2520                                         /* op1 = target JavaVM pc, val.i = constant   */
2521
2522                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2523                         break;
2524
2525                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2526                                         /* op1 = target JavaVM pc, val.i = constant   */
2527
2528                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2529                         break;
2530
2531                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2532                                         /* op1 = target JavaVM pc, val.i = constant   */
2533
2534                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2535                         break;
2536
2537                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2538                                         /* op1 = target JavaVM pc, val.i = constant   */
2539
2540                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2541                         break;
2542
2543                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2544                                         /* op1 = target JavaVM pc, val.i = constant   */
2545
2546                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2547                         break;
2548
2549                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2550                                         /* op1 = target JavaVM pc, val.i = constant   */
2551
2552                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2553                         break;
2554
2555                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2556                                         /* op1 = target JavaVM pc, val.l = constant   */
2557
2558                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2559                         break;
2560
2561                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2562                                         /* op1 = target JavaVM pc, val.l = constant   */
2563
2564                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2565                         break;
2566
2567                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2568                                         /* op1 = target JavaVM pc, val.l = constant   */
2569
2570                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2571                         break;
2572
2573                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2574                                         /* op1 = target JavaVM pc, val.l = constant   */
2575
2576                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2577                         break;
2578
2579                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2580                                         /* op1 = target JavaVM pc, val.l = constant   */
2581
2582                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2583                         break;
2584
2585                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2586                                         /* op1 = target JavaVM pc, val.l = constant   */
2587
2588                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2589                         break;
2590
2591                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2592                                         /* op1 = target JavaVM pc                     */
2593
2594                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2595                         break;
2596
2597                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2598                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2599
2600                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2601                         break;
2602
2603                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2604                                         /* op1 = target JavaVM pc                     */
2605
2606                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2607                         break;
2608
2609                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2610                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2611
2612                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2613                         break;
2614
2615                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2616                                         /* op1 = target JavaVM pc                     */
2617
2618                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2619                         break;
2620
2621                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2622                                     /* op1 = target JavaVM pc                     */
2623
2624                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2625                         break;
2626
2627                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2628                                         /* op1 = target JavaVM pc                     */
2629
2630                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2631                         break;
2632
2633                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2634                                 /* op1 = target JavaVM pc                     */
2635
2636                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2637                         break;
2638
2639                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2640                                         /* op1 = target JavaVM pc                     */
2641
2642                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2643                         break;
2644
2645                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2646                                         /* op1 = target JavaVM pc                     */
2647
2648                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2649                         break;
2650
2651                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2652                                         /* op1 = target JavaVM pc                     */
2653
2654                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2655                         break;
2656
2657                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2658                                     /* op1 = target JavaVM pc                     */
2659
2660                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2661                         break;
2662
2663                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2664
2665                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2666                         break;
2667
2668                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2669                                         /* val.i = constant                           */
2670
2671                         var_to_reg_int(s1, src, REG_ITMP1);
2672                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2673                         s3 = iptr->val.i;
2674                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2675                                 if (s1 == d) {
2676                                         M_INTMOVE(s1, REG_ITMP1);
2677                                         s1 = REG_ITMP1;
2678                                 }
2679                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2680                         }
2681                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2682                         x86_64_testl_reg_reg(cd, s1, s1);
2683                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2684                         store_reg_to_var_int(iptr->dst, d);
2685                         break;
2686
2687                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2688                                         /* val.i = constant                           */
2689
2690                         var_to_reg_int(s1, src, REG_ITMP1);
2691                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2692                         s3 = iptr->val.i;
2693                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2694                                 if (s1 == d) {
2695                                         M_INTMOVE(s1, REG_ITMP1);
2696                                         s1 = REG_ITMP1;
2697                                 }
2698                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2699                         }
2700                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2701                         x86_64_testl_reg_reg(cd, s1, s1);
2702                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2703                         store_reg_to_var_int(iptr->dst, d);
2704                         break;
2705
2706                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2707                                         /* val.i = constant                           */
2708
2709                         var_to_reg_int(s1, src, REG_ITMP1);
2710                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2711                         s3 = iptr->val.i;
2712                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2713                                 if (s1 == d) {
2714                                         M_INTMOVE(s1, REG_ITMP1);
2715                                         s1 = REG_ITMP1;
2716                                 }
2717                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2718                         }
2719                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2720                         x86_64_testl_reg_reg(cd, s1, s1);
2721                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2722                         store_reg_to_var_int(iptr->dst, d);
2723                         break;
2724
2725                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2726                                         /* val.i = constant                           */
2727
2728                         var_to_reg_int(s1, src, REG_ITMP1);
2729                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2730                         s3 = iptr->val.i;
2731                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2732                                 if (s1 == d) {
2733                                         M_INTMOVE(s1, REG_ITMP1);
2734                                         s1 = REG_ITMP1;
2735                                 }
2736                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2737                         }
2738                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2739                         x86_64_testl_reg_reg(cd, s1, s1);
2740                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2741                         store_reg_to_var_int(iptr->dst, d);
2742                         break;
2743
2744                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2745                                         /* val.i = constant                           */
2746
2747                         var_to_reg_int(s1, src, REG_ITMP1);
2748                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2749                         s3 = iptr->val.i;
2750                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2751                                 if (s1 == d) {
2752                                         M_INTMOVE(s1, REG_ITMP1);
2753                                         s1 = REG_ITMP1;
2754                                 }
2755                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2756                         }
2757                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2758                         x86_64_testl_reg_reg(cd, s1, s1);
2759                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2760                         store_reg_to_var_int(iptr->dst, d);
2761                         break;
2762
2763                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2764                                         /* val.i = constant                           */
2765
2766                         var_to_reg_int(s1, src, REG_ITMP1);
2767                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2768                         s3 = iptr->val.i;
2769                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2770                                 if (s1 == d) {
2771                                         M_INTMOVE(s1, REG_ITMP1);
2772                                         s1 = REG_ITMP1;
2773                                 }
2774                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2775                         }
2776                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2777                         x86_64_testl_reg_reg(cd, s1, s1);
2778                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2779                         store_reg_to_var_int(iptr->dst, d);
2780                         break;
2781
2782
2783                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2784                 case ICMD_LRETURN:
2785                 case ICMD_ARETURN:
2786
2787                         var_to_reg_int(s1, src, REG_RESULT);
2788                         M_INTMOVE(s1, REG_RESULT);
2789
2790                         goto nowperformreturn;
2791
2792                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2793                 case ICMD_DRETURN:
2794
2795                         var_to_reg_flt(s1, src, REG_FRESULT);
2796                         M_FLTMOVE(s1, REG_FRESULT);
2797
2798                         goto nowperformreturn;
2799
2800                 case ICMD_RETURN:      /* ...  ==> ...                                */
2801
2802 nowperformreturn:
2803                         {
2804                         s4 i, p;
2805
2806                         p = parentargs_base;
2807
2808                         /* call trace function */
2809                         if (runverbose) {
2810                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2811
2812                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2813                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2814
2815                                 x86_64_mov_imm_reg(cd, (s8) m, rd->argintregs[0]);
2816                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2817                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2818                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2819
2820                                 x86_64_mov_imm_reg(cd, (s8) builtin_displaymethodstop, REG_ITMP1);
2821                                 x86_64_call_reg(cd, REG_ITMP1);
2822
2823                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2824                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2825
2826                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2827                         }
2828
2829 #if defined(USE_THREADS)
2830                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2831                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2832
2833                                 /* we need to save the proper return value */
2834                                 switch (iptr->opc) {
2835                                 case ICMD_IRETURN:
2836                                 case ICMD_ARETURN:
2837                                 case ICMD_LRETURN:
2838                                         x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->maxmemuse * 8);
2839                                         break;
2840                                 case ICMD_FRETURN:
2841                                 case ICMD_DRETURN:
2842                                         x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->maxmemuse * 8);
2843                                         break;
2844                                 }
2845
2846                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2847                                 x86_64_call_reg(cd, REG_ITMP1);
2848
2849                                 /* and now restore the proper return value */
2850                                 switch (iptr->opc) {
2851                                 case ICMD_IRETURN:
2852                                 case ICMD_ARETURN:
2853                                 case ICMD_LRETURN:
2854                                         x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_RESULT);
2855                                         break;
2856                                 case ICMD_FRETURN:
2857                                 case ICMD_DRETURN:
2858                                         x86_64_movq_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_FRESULT);
2859                                         break;
2860                                 }
2861                         }
2862 #endif
2863
2864                         /* restore saved registers                                        */
2865                         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
2866                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
2867                         }
2868                         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
2869                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
2870                         }
2871
2872                         /* deallocate stack                                               */
2873                         if (parentargs_base) {
2874                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2875                         }
2876
2877                         x86_64_ret(cd);
2878                         ALIGNCODENOP;
2879                         }
2880                         break;
2881
2882
2883                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2884                         {
2885                                 s4 i, l, *s4ptr;
2886                                 void **tptr;
2887
2888                                 tptr = (void **) iptr->target;
2889
2890                                 s4ptr = iptr->val.a;
2891                                 l = s4ptr[1];                          /* low     */
2892                                 i = s4ptr[2];                          /* high    */
2893
2894                                 var_to_reg_int(s1, src, REG_ITMP1);
2895                                 M_INTMOVE(s1, REG_ITMP1);
2896                                 if (l != 0) {
2897                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2898                                 }
2899                                 i = i - l + 1;
2900
2901                 /* range check */
2902                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2903                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2904
2905                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2906                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2907
2908                                 /* build jump table top down and use address of lowest entry */
2909
2910                 /* s4ptr += 3 + i; */
2911                                 tptr += i;
2912
2913                                 while (--i >= 0) {
2914                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
2915                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2916                                         --tptr;
2917                                 }
2918
2919                                 /* length of dataseg after last dseg_addtarget is used by load */
2920
2921                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2922                                 dseg_adddata(cd, cd->mcodeptr);
2923                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2924                                 x86_64_jmp_reg(cd, REG_ITMP1);
2925                                 ALIGNCODENOP;
2926                         }
2927                         break;
2928
2929
2930                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2931                         {
2932                                 s4 i, l, val, *s4ptr;
2933                                 void **tptr;
2934
2935                                 tptr = (void **) iptr->target;
2936
2937                                 s4ptr = iptr->val.a;
2938                                 l = s4ptr[0];                          /* default  */
2939                                 i = s4ptr[1];                          /* count    */
2940
2941                                 MCODECHECK((i<<2)+8);
2942                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
2943                                 while (--i >= 0) {
2944                                         s4ptr += 2;
2945                                         ++tptr;
2946
2947                                         val = s4ptr[0];
2948                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
2949                                         x86_64_jcc(cd, X86_64_CC_E, 0);
2950                                         /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
2951                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2952                                 }
2953
2954                                 x86_64_jmp_imm(cd, 0);
2955                                 /* codegen_addreference(cd, BlockPtrOfPC(l), cd->mcodeptr); */
2956
2957                                 tptr = (void **) iptr->target;
2958                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2959
2960                                 ALIGNCODENOP;
2961                         }
2962                         break;
2963
2964
2965                 case ICMD_BUILTIN3:     /* ..., arg1, arg2, arg3 ==> ...              */
2966                                         /* op1 = return type, val.a = function pointer*/
2967                         s3 = 3;
2968                         goto gen_method;
2969
2970                 case ICMD_BUILTIN2:     /* ..., arg1, arg2 ==> ...                    */
2971                                         /* op1 = return type, val.a = function pointer*/
2972                         s3 = 2;
2973                         goto gen_method;
2974
2975                 case ICMD_BUILTIN1:     /* ..., arg1 ==> ...                          */
2976                                         /* op1 = return type, val.a = function pointer*/
2977                         s3 = 1;
2978                         goto gen_method;
2979
2980                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2981                                         /* op1 = arg count, val.a = method pointer    */
2982
2983                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2984                                         /* op1 = arg count, val.a = method pointer    */
2985
2986                 case ICMD_INVOKEVIRTUAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2987                                         /* op1 = arg count, val.a = method pointer    */
2988
2989                 case ICMD_INVOKEINTERFACE:/*.., objectref, [arg1, [arg2 ...]] ==> ... */
2990                                         /* op1 = arg count, val.a = method pointer    */
2991
2992                         s3 = iptr->op1;
2993
2994 gen_method: {
2995                         methodinfo   *lm;
2996                         classinfo    *ci;
2997                         stackptr      tmpsrc;
2998                         s4 iarg, farg;
2999
3000                         MCODECHECK((s3 << 1) + 64);
3001
3002                         tmpsrc = src;
3003                         s2 = s3;
3004                         iarg = 0;
3005                         farg = 0;
3006
3007                         /* copy arguments to registers or stack location ******************/
3008
3009                         /* count integer and float arguments */
3010
3011                         for (; --s3 >= 0; src = src->prev) {
3012                                 IS_INT_LNG_TYPE(src->type) ? iarg++ : farg++;
3013                         }
3014
3015                         src = tmpsrc;
3016                         s3 = s2;
3017
3018                         /* calculate amount of arguments to be on stack */
3019
3020                         s2 = (iarg > INT_ARG_CNT) ? iarg - INT_ARG_CNT : 0 +
3021                                 (farg > FLT_ARG_CNT) ? farg - FLT_ARG_CNT : 0;
3022
3023                         for (; --s3 >= 0; src = src->prev) {
3024                                 /* decrement the current argument type */
3025                                 IS_INT_LNG_TYPE(src->type) ? iarg-- : farg--;
3026
3027                                 if (src->varkind == ARGVAR) {
3028                                         if (IS_INT_LNG_TYPE(src->type)) {
3029                                                 if (iarg >= INT_ARG_CNT) {
3030                                                         s2--;
3031                                                 }
3032                                         } else {
3033                                                 if (farg >= FLT_ARG_CNT) {
3034                                                         s2--;
3035                                                 }
3036                                         }
3037                                         continue;
3038                                 }
3039
3040                                 if (IS_INT_LNG_TYPE(src->type)) {
3041                                         if (iarg < INT_ARG_CNT) {
3042                                                 s1 = rd->argintregs[iarg];
3043                                                 var_to_reg_int(d, src, s1);
3044                                                 M_INTMOVE(d, s1);
3045
3046                                         } else {
3047                                                 var_to_reg_int(d, src, REG_ITMP1);
3048                                                 s2--;
3049                                                 x86_64_mov_reg_membase(cd, d, REG_SP, s2 * 8);
3050                                         }
3051
3052                                 } else {
3053                                         if (farg < FLT_ARG_CNT) {
3054                                                 s1 = rd->argfltregs[farg];
3055                                                 var_to_reg_flt(d, src, s1);
3056                                                 M_FLTMOVE(d, s1);
3057
3058                                         } else {
3059                                                 var_to_reg_flt(d, src, REG_FTMP1);
3060                                                 s2--;
3061                                                 x86_64_movq_reg_membase(cd, d, REG_SP, s2 * 8);
3062                                         }
3063                                 }
3064                         } /* end of for */
3065
3066                         lm = iptr->val.a;
3067                         switch (iptr->opc) {
3068                                 case ICMD_BUILTIN3:
3069                                 case ICMD_BUILTIN2:
3070                                 case ICMD_BUILTIN1:
3071
3072                                         a = (s8) lm;
3073                                         d = iptr->op1;
3074
3075                                         x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3076                                         x86_64_call_reg(cd, REG_ITMP1);
3077                                         break;
3078
3079                                 case ICMD_INVOKESTATIC:
3080
3081                                         a = (s8) lm->stubroutine;
3082                                         d = lm->returntype;
3083
3084                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3085                                         x86_64_call_reg(cd, REG_ITMP2);
3086                                         break;
3087
3088                                 case ICMD_INVOKESPECIAL:
3089
3090                                         a = (s8) lm->stubroutine;
3091                                         d = lm->returntype;
3092
3093                                         gen_nullptr_check(rd->argintregs[0]);    /* first argument contains pointer */
3094                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); /* access memory for hardware nullptr */
3095                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3096                                         x86_64_call_reg(cd, REG_ITMP2);
3097                                         break;
3098
3099                                 case ICMD_INVOKEVIRTUAL:
3100
3101                                         d = lm->returntype;
3102
3103                                         gen_nullptr_check(rd->argintregs[0]);
3104                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3105                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, OFFSET(vftbl_t, table[0]) + sizeof(methodptr) * lm->vftblindex, REG_ITMP1);
3106                                         x86_64_call_reg(cd, REG_ITMP1);
3107                                         break;
3108
3109                                 case ICMD_INVOKEINTERFACE:
3110
3111                                         ci = lm->class;
3112                                         d = lm->returntype;
3113
3114                                         gen_nullptr_check(rd->argintregs[0]);
3115                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3116                                         x86_64_mov_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, interfacetable[0]) - sizeof(methodptr) * ci->index, REG_ITMP2);
3117                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, sizeof(methodptr) * (lm - ci->methods), REG_ITMP1);
3118                                         x86_64_call_reg(cd, REG_ITMP1);
3119                                         break;
3120
3121                                 default:
3122                                         d = 0;
3123                                         error("Unkown ICMD-Command: %d", iptr->opc);
3124                                 }
3125
3126                         /* d contains return type */
3127
3128                         if (d != TYPE_VOID) {
3129                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3130                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3131                                         M_INTMOVE(REG_RESULT, s1);
3132                                         store_reg_to_var_int(iptr->dst, s1);
3133
3134                                 } else {
3135                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3136                                         M_FLTMOVE(REG_FRESULT, s1);
3137                                         store_reg_to_var_flt(iptr->dst, s1);
3138                                 }
3139                         }
3140                         }
3141                         break;
3142
3143
3144                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3145
3146                                       /* op1:   0 == array, 1 == class                */
3147                                       /* val.a: (classinfo*) superclass               */
3148
3149 /*          superclass is an interface:
3150  *
3151  *          return (sub != NULL) &&
3152  *                 (sub->vftbl->interfacetablelength > super->index) &&
3153  *                 (sub->vftbl->interfacetable[-super->index] != NULL);
3154  *
3155  *          superclass is a class:
3156  *
3157  *          return ((sub != NULL) && (0
3158  *                  <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3159  *                  super->vftbl->diffvall));
3160  */
3161
3162                         {
3163                         classinfo *super = (classinfo*) iptr->val.a;
3164
3165 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3166             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3167 #endif
3168
3169                         var_to_reg_int(s1, src, REG_ITMP1);
3170                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3171                         if (s1 == d) {
3172                                 M_INTMOVE(s1, REG_ITMP1);
3173                                 s1 = REG_ITMP1;
3174                         }
3175                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3176                         if (iptr->op1) {                               /* class/interface */
3177                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3178                                         x86_64_test_reg_reg(cd, s1, s1);
3179
3180                                         /* TODO: clean up this calculation */
3181                                         a = 3;    /* mov_membase_reg */
3182                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3183
3184                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3185                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3186
3187                                         a += 3;    /* sub */
3188                                         CALCIMMEDIATEBYTES(a, super->index);
3189
3190                                         a += 3;    /* test */
3191
3192                                         a += 6;    /* jcc */
3193                                         a += 3;    /* mov_membase_reg */
3194                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3195
3196                                         a += 3;    /* test */
3197                                         a += 4;    /* setcc */
3198
3199                                         x86_64_jcc(cd, X86_64_CC_E, a);
3200
3201                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3202                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3203                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3204                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3205
3206                                         /* TODO: clean up this calculation */
3207                                         a = 0;
3208                                         a += 3;    /* mov_membase_reg */
3209                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3210
3211                                         a += 3;    /* test */
3212                                         a += 4;    /* setcc */
3213
3214                                         x86_64_jcc(cd, X86_64_CC_LE, a);
3215                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP1);
3216                                         x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3217                                         x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3218
3219                                 } else {                                   /* class           */
3220                                         x86_64_test_reg_reg(cd, s1, s1);
3221
3222                                         /* TODO: clean up this calculation */
3223                                         a = 3;    /* mov_membase_reg */
3224                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3225
3226                                         a += 10;   /* mov_imm_reg */
3227
3228                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3229                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3230
3231                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3232                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3233
3234                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3235                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3236
3237                                         a += 3;    /* sub */
3238                                         a += 3;    /* xor */
3239                                         a += 3;    /* cmp */
3240                                         a += 4;    /* setcc */
3241
3242                                         x86_64_jcc(cd, X86_64_CC_E, a);
3243
3244                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3245                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3246 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3247                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3248 #endif
3249                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3250                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3251                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3252 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3253                     codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3254 #endif
3255                                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3256                                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3257                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3258                                         x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3259                                 }
3260                         }
3261                         else
3262                                 panic("internal error: no inlined array instanceof");
3263                         }
3264                         store_reg_to_var_int(iptr->dst, d);
3265                         break;
3266
3267                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3268
3269                                       /* op1:   0 == array, 1 == class                */
3270                                       /* val.a: (classinfo*) superclass               */
3271
3272 /*          superclass is an interface:
3273  *
3274  *          OK if ((sub == NULL) ||
3275  *                 (sub->vftbl->interfacetablelength > super->index) &&
3276  *                 (sub->vftbl->interfacetable[-super->index] != NULL));
3277  *
3278  *          superclass is a class:
3279  *
3280  *          OK if ((sub == NULL) || (0
3281  *                 <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3282  *                 super->vftbl->diffvall));
3283  */
3284
3285                         {
3286                         classinfo *super = (classinfo*) iptr->val.a;
3287
3288 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3289             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3290 #endif
3291                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3292                         var_to_reg_int(s1, src, d);
3293                         if (iptr->op1) {                               /* class/interface */
3294                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3295                                         x86_64_test_reg_reg(cd, s1, s1);
3296
3297                                         /* TODO: clean up this calculation */
3298                                         a = 3;    /* mov_membase_reg */
3299                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3300
3301                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3302                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3303
3304                                         a += 3;    /* sub */
3305                                         CALCIMMEDIATEBYTES(a, super->index);
3306
3307                                         a += 3;    /* test */
3308                                         a += 6;    /* jcc */
3309
3310                                         a += 3;    /* mov_membase_reg */
3311                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3312
3313                                         a += 3;    /* test */
3314                                         a += 6;    /* jcc */
3315
3316                                         x86_64_jcc(cd, X86_64_CC_E, a);
3317
3318                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3319                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3320                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3321                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3322                                         x86_64_jcc(cd, X86_64_CC_LE, 0);
3323                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3324                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP2);
3325                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3326                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3327                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3328
3329                                 } else {                                     /* class           */
3330                                         x86_64_test_reg_reg(cd, s1, s1);
3331
3332                                         /* TODO: clean up this calculation */
3333                                         a = 3;    /* mov_membase_reg */
3334                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3335                                         a += 10;   /* mov_imm_reg */
3336                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3337                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3338
3339                                         if (d != REG_ITMP3) {
3340                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3341                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3342                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3343                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3344                                                 a += 3;    /* sub */
3345
3346                                         } else {
3347                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3348                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3349                                                 a += 3;    /* sub */
3350                                                 a += 10;   /* mov_imm_reg */
3351                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3352                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3353                                         }
3354
3355                                         a += 3;    /* cmp */
3356                                         a += 6;    /* jcc */
3357
3358                                         x86_64_jcc(cd, X86_64_CC_E, a);
3359
3360                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3361                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3362 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3363                     codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3364 #endif
3365                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3366                                         if (d != REG_ITMP3) {
3367                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3368                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3369 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3370                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3371 #endif
3372                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3373
3374                                         } else {
3375                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP2);
3376                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3377                                                 x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3378                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3379 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3380                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3381 #endif
3382                                         }
3383                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3384                                         x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3385                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3386                                 }
3387
3388                         } else
3389                                 panic("internal error: no inlined array checkcast");
3390                         }
3391                         M_INTMOVE(s1, d);
3392                         store_reg_to_var_int(iptr->dst, d);
3393                         break;
3394
3395                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3396
3397                         if (src->flags & INMEMORY) {
3398                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3399
3400                         } else {
3401                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3402                         }
3403                         x86_64_jcc(cd, X86_64_CC_L, 0);
3404                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3405                         break;
3406
3407                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3408
3409                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3410                         x86_64_jcc(cd, X86_64_CC_E, 0);
3411                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3412                         break;
3413
3414                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3415                                          /* op1 = dimension, val.a = array descriptor */
3416
3417                         /* check for negative sizes and copy sizes to stack if necessary  */
3418
3419                         MCODECHECK((iptr->op1 << 1) + 64);
3420
3421                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3422                                 var_to_reg_int(s2, src, REG_ITMP1);
3423                                 x86_64_testl_reg_reg(cd, s2, s2);
3424                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3425                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3426
3427                                 /* copy sizes to stack (argument numbers >= INT_ARG_CNT)      */
3428
3429                                 if (src->varkind != ARGVAR) {
3430                                         x86_64_mov_reg_membase(cd, s2, REG_SP, (s1 + INT_ARG_CNT) * 8);
3431                                 }
3432                         }
3433
3434                         /* a0 = dimension count */
3435                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3436
3437                         /* a1 = arraydescriptor */
3438                         x86_64_mov_imm_reg(cd, (s8) iptr->val.a, rd->argintregs[1]);
3439
3440                         /* a2 = pointer to dimensions = stack pointer */
3441                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3442
3443                         x86_64_mov_imm_reg(cd, (s8) builtin_nmultianewarray, REG_ITMP1);
3444                         x86_64_call_reg(cd, REG_ITMP1);
3445
3446                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3447                         M_INTMOVE(REG_RESULT, s1);
3448                         store_reg_to_var_int(iptr->dst, s1);
3449                         break;
3450
3451                 default:
3452                         throw_cacao_exception_exit(string_java_lang_InternalError,
3453                                                                            "Unknown ICMD %d", iptr->opc);
3454         } /* switch */
3455
3456         } /* for instruction */
3457
3458         /* copy values to interface registers */
3459
3460         src = bptr->outstack;
3461         len = bptr->outdepth;
3462         MCODECHECK(64 + len);
3463         while (src) {
3464                 len--;
3465                 if ((src->varkind != STACKVAR)) {
3466                         s2 = src->type;
3467                         if (IS_FLT_DBL_TYPE(s2)) {
3468                                 var_to_reg_flt(s1, src, REG_FTMP1);
3469                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3470                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3471
3472                                 } else {
3473                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3474                                 }
3475
3476                         } else {
3477                                 var_to_reg_int(s1, src, REG_ITMP1);
3478                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3479                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3480
3481                                 } else {
3482                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3483                                 }
3484                         }
3485                 }
3486                 src = src->prev;
3487         }
3488         } /* if (bptr -> flags >= BBREACHED) */
3489         } /* for basic block */
3490
3491         {
3492
3493         /* generate bound check stubs */
3494
3495         u1 *xcodeptr = NULL;
3496         branchref *bref;
3497
3498         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3499                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3500                                   bref->branchpos,
3501                                                   cd->mcodeptr - cd->mcodebase);
3502
3503                 MCODECHECK(50);
3504
3505                 /* move index register into REG_ITMP1 */
3506                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);              /* 3 bytes  */
3507
3508                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                  /* 10 bytes */
3509                 dseg_adddata(cd, cd->mcodeptr);
3510                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3511                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3512
3513                 if (xcodeptr != NULL) {
3514                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3515
3516                 } else {
3517                         xcodeptr = cd->mcodeptr;
3518
3519                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3520                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3521
3522                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3523                         x86_64_mov_imm_reg(cd, (s8) new_arrayindexoutofboundsexception, REG_ITMP3);
3524                         x86_64_call_reg(cd, REG_ITMP3);
3525
3526                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3527                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3528
3529                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3530                         x86_64_jmp_reg(cd, REG_ITMP3);
3531                 }
3532         }
3533
3534         /* generate negative array size check stubs */
3535
3536         xcodeptr = NULL;
3537
3538         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3539                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3540                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3541                                                           bref->branchpos,
3542                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3543                         continue;
3544                 }
3545
3546                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3547                                   bref->branchpos,
3548                                                   cd->mcodeptr - cd->mcodebase);
3549
3550                 MCODECHECK(50);
3551
3552                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                         /* 10 bytes */
3553                 dseg_adddata(cd, cd->mcodeptr);
3554                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3555                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);     /* 3 bytes  */
3556
3557                 if (xcodeptr != NULL) {
3558                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3559
3560                 } else {
3561                         xcodeptr = cd->mcodeptr;
3562
3563                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3564                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3565
3566                         x86_64_mov_imm_reg(cd, (s8) new_negativearraysizeexception, REG_ITMP3);
3567                         x86_64_call_reg(cd, REG_ITMP3);
3568
3569                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3570                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3571
3572                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3573                         x86_64_jmp_reg(cd, REG_ITMP3);
3574                 }
3575         }
3576
3577         /* generate cast check stubs */
3578
3579         xcodeptr = NULL;
3580
3581         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3582                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3583                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3584                                                           bref->branchpos,
3585                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3586                         continue;
3587                 }
3588
3589                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3590                                   bref->branchpos,
3591                                                   cd->mcodeptr - cd->mcodebase);
3592
3593                 MCODECHECK(50);
3594
3595                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3596                 dseg_adddata(cd, cd->mcodeptr);
3597                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);     /* 10 bytes */
3598                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3599
3600                 if (xcodeptr != NULL) {
3601                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3602
3603                 } else {
3604                         xcodeptr = cd->mcodeptr;
3605
3606                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3607                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3608
3609                         x86_64_mov_imm_reg(cd, (s8) new_classcastexception, REG_ITMP3);
3610                         x86_64_call_reg(cd, REG_ITMP3);
3611
3612                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3613                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3614
3615                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3616                         x86_64_jmp_reg(cd, REG_ITMP3);
3617                 }
3618         }
3619
3620         /* generate divide by zero check stubs */
3621
3622         xcodeptr = NULL;
3623
3624         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3625                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3626                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3627                                                           bref->branchpos,
3628                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3629                         continue;
3630                 }
3631
3632                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3633                                   bref->branchpos,
3634                                                   cd->mcodeptr - cd->mcodebase);
3635
3636                 MCODECHECK(50);
3637
3638                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3639                 dseg_adddata(cd, cd->mcodeptr);
3640                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);      /* 10 bytes */
3641                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3642
3643                 if (xcodeptr != NULL) {
3644                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3645
3646                 } else {
3647                         xcodeptr = cd->mcodeptr;
3648
3649                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3650                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3651
3652                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
3653                         x86_64_call_reg(cd, REG_ITMP3);
3654
3655                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3656                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3657
3658                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3659                         x86_64_jmp_reg(cd, REG_ITMP3);
3660                 }
3661         }
3662
3663         /* generate exception check stubs */
3664
3665         xcodeptr = NULL;
3666
3667         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3668                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3669                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3670                                                           bref->branchpos,
3671                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3672                         continue;
3673                 }
3674
3675                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3676                                   bref->branchpos,
3677                                                   cd->mcodeptr - cd->mcodebase);
3678
3679                 MCODECHECK(50);
3680
3681                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3682                 dseg_adddata(cd, cd->mcodeptr);
3683                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3684                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3685
3686                 if (xcodeptr != NULL) {
3687                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3688
3689                 } else {
3690                         xcodeptr = cd->mcodeptr;
3691
3692 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3693                         x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3694                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0);
3695                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3696                         x86_64_call_reg(cd, REG_ITMP1);
3697                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3698                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3699                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3700                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
3701                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
3702 #else
3703                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
3704                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
3705                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
3706 #endif
3707
3708                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3709                         x86_64_jmp_reg(cd, REG_ITMP3);
3710                 }
3711         }
3712
3713         /* generate null pointer check stubs */
3714
3715         xcodeptr = NULL;
3716
3717         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3718                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3719                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3720                                                           bref->branchpos,
3721                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3722                         continue;
3723                 }
3724
3725                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3726                                   bref->branchpos,
3727                                                   cd->mcodeptr - cd->mcodebase);
3728
3729                 MCODECHECK(50);
3730
3731                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3732                 dseg_adddata(cd, cd->mcodeptr);
3733                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3734                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3735
3736                 if (xcodeptr != NULL) {
3737                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3738
3739                 } else {
3740                         xcodeptr = cd->mcodeptr;
3741
3742                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3743                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3744
3745                         x86_64_mov_imm_reg(cd, (s8) new_nullpointerexception, REG_ITMP3);
3746                         x86_64_call_reg(cd, REG_ITMP3);
3747
3748                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3749                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3750
3751                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3752                         x86_64_jmp_reg(cd, REG_ITMP3);
3753                 }
3754         }
3755
3756         /* generate put/getstatic stub call code */
3757
3758         {
3759                 clinitref   *cref;
3760                 codegendata *tmpcd;
3761                 u1           xmcode;
3762                 u4           mcode;
3763
3764                 tmpcd = DNEW(codegendata);
3765
3766                 for (cref = cd->clinitrefs; cref != NULL; cref = cref->next) {
3767                         /* Get machine code which is patched back in later. A             */
3768                         /* `call rel32' is 5 bytes long.                                  */
3769                         xcodeptr = cd->mcodebase + cref->branchpos;
3770                         xmcode = *xcodeptr;
3771                         mcode = *((u4 *) (xcodeptr + 1));
3772
3773                         MCODECHECK(50);
3774
3775                         /* patch in `call rel32' to call the following code               */
3776                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
3777                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
3778
3779                         /* Save current stack pointer into a temporary register.          */
3780                         x86_64_mov_reg_reg(cd, REG_SP, REG_ITMP1);
3781
3782                         /* Push machine code bytes to patch onto the stack.               */
3783                         x86_64_push_imm(cd, (u1) xmcode);
3784                         x86_64_push_imm(cd, (u4) mcode);
3785
3786                         x86_64_push_imm(cd, (u8) cref->class);
3787
3788                         /* Push previously saved stack pointer onto stack.                */
3789                         x86_64_push_reg(cd, REG_ITMP1);
3790
3791                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP1);
3792                         x86_64_jmp_reg(cd, REG_ITMP1);
3793                 }
3794         }
3795         }
3796
3797         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
3798 }
3799
3800
3801 /* function createcompilerstub *************************************************
3802
3803         creates a stub routine which calls the compiler
3804
3805 *******************************************************************************/
3806
3807 #define COMPSTUBSIZE 23
3808
3809 u1 *createcompilerstub(methodinfo *m)
3810 {
3811         u1 *s = CNEW(u1, COMPSTUBSIZE);     /* memory to hold the stub            */
3812         codegendata *cd;
3813         s4 dumpsize;
3814
3815         /* mark start of dump memory area */
3816
3817         dumpsize = dump_size();
3818
3819         cd = DNEW(codegendata);
3820         cd->mcodeptr = s;
3821
3822         /* code for the stub */
3823         x86_64_mov_imm_reg(cd, (u8) m, REG_ITMP1); /* pass method to compiler     */
3824         x86_64_mov_imm_reg(cd, (u8) asm_call_jit_compiler, REG_ITMP3);/* load address */
3825         x86_64_jmp_reg(cd, REG_ITMP3);      /* jump to compiler                   */
3826
3827 #if defined(STATISTICS)
3828         if (opt_stat)
3829                 count_cstub_len += COMPSTUBSIZE;
3830 #endif
3831
3832         /* release dump area */
3833
3834         dump_release(dumpsize);
3835
3836         return s;
3837 }
3838
3839
3840 /* function removecompilerstub *************************************************
3841
3842      deletes a compilerstub from memory  (simply by freeing it)
3843
3844 *******************************************************************************/
3845
3846 void removecompilerstub(u1 *stub)
3847 {
3848         CFREE(stub, COMPSTUBSIZE);
3849 }
3850
3851
3852 /* function: createnativestub **************************************************
3853
3854         creates a stub routine which calls a native method
3855
3856 *******************************************************************************/
3857
3858 /* #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3859 /* static java_objectheader **(*callgetexceptionptrptr)() = builtin_get_exceptionptrptr; */
3860 /* #endif */
3861
3862 #define NATIVESTUBSIZE    700           /* keep this size high enough!        */
3863
3864 u1 *createnativestub(functionptr f, methodinfo *m)
3865 {
3866         u1                 *s;              /* pointer to stub memory             */
3867         codegendata        *cd;
3868         registerdata       *rd;
3869         t_inlining_globals *id;
3870         s4                  dumpsize;
3871         s4                  stackframesize; /* size of stackframe if needed       */
3872         u1                 *tptr;
3873         s4                  iargs;          /* count of integer arguments         */
3874         s4                  fargs;          /* count of float arguments           */
3875         s4                  i;              /* counter                            */
3876
3877         void **callAddrPatchPos=0;
3878         u1 *jmpInstrPos=0;
3879         void **jmpInstrPatchPos=0;
3880
3881         /* initialize variables */
3882
3883         iargs = 0;
3884         fargs = 0;
3885
3886         /* mark start of dump memory area */
3887
3888         dumpsize = dump_size();
3889
3890         cd = DNEW(codegendata);
3891         rd = DNEW(registerdata);
3892         id = DNEW(t_inlining_globals);
3893
3894         /* setup registers before using it */
3895
3896         inlining_setup(m, id);
3897         reg_setup(m, rd, id);
3898
3899         /* set paramcount and paramtypes      */
3900
3901         descriptor2types(m);
3902
3903         /* count integer and float arguments */
3904
3905         tptr = m->paramtypes;
3906         for (i = 0; i < m->paramcount; i++) {
3907                 IS_INT_LNG_TYPE(*tptr++) ? iargs++ : fargs++;
3908         }
3909
3910         s = CNEW(u1, NATIVESTUBSIZE);       /* memory to hold the stub            */
3911
3912         /* set some required varibles which are normally set by codegen_setup */
3913         cd->mcodebase = s;
3914         cd->mcodeptr = s;
3915         cd->clinitrefs = NULL;
3916
3917         /* if function is static, check for initialized */
3918
3919         if ((m->flags & ACC_STATIC) && !m->class->initialized) {
3920                 codegen_addclinitref(cd, cd->mcodeptr, m->class);
3921         }
3922
3923         if (runverbose) {
3924                 s4 l, s1;
3925
3926                 x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3927
3928                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
3929                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
3930                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
3931                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
3932                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
3933                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
3934
3935                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
3936                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
3937                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
3938                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
3939                 x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8);
3940                 x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8);
3941                 x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8);
3942                 x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8);
3943
3944                 /* show integer hex code for float arguments */
3945
3946                 for (i = 0, l = 0; i < m->paramcount; i++) {
3947                         if (IS_FLT_DBL_TYPE(m->paramtypes[i])) {
3948                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= i; s1--) {
3949                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
3950                                 }
3951
3952                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[i]);
3953                                 l++;
3954                         }
3955                 }
3956
3957                 x86_64_mov_imm_reg(cd, (u8) m, REG_ITMP1);
3958                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
3959                 x86_64_mov_imm_reg(cd, (u8) builtin_trace_args, REG_ITMP1);
3960                 x86_64_call_reg(cd, REG_ITMP1);
3961
3962                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
3963                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
3964                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
3965                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
3966                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
3967                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
3968
3969                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
3970                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
3971                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
3972                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
3973                 x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]);
3974                 x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]);
3975                 x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]);
3976                 x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]);
3977
3978                 x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3979         }
3980
3981 #if !defined(STATIC_CLASSPATH)
3982         /* call method to resolve native function if needed */
3983         if (f == NULL) {
3984                 x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3985
3986                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
3987                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
3988                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
3989                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
3990                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
3991                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
3992
3993                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
3994                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
3995                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
3996                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
3997                 x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8);
3998                 x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8);
3999                 x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8);
4000                 x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8);
4001
4002                 /* needed to patch a jump over this block */
4003                 x86_64_jmp_imm(cd, 0);
4004                 jmpInstrPos = cd->mcodeptr - 4;
4005
4006                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
4007
4008                 x86_64_mov_imm_reg(cd, 0, rd->argintregs[1]);
4009                 callAddrPatchPos = cd->mcodeptr - 8; /* at this position the place is specified where the native function adress should be patched into*/
4010
4011                 x86_64_mov_imm_reg(cd, 0, rd->argintregs[2]);
4012                 jmpInstrPatchPos = cd->mcodeptr - 8;
4013
4014                 x86_64_mov_imm_reg(cd, jmpInstrPos, rd->argintregs[3]);
4015
4016                 x86_64_mov_imm_reg(cd, (u8) codegen_resolve_native, REG_ITMP1);
4017                 x86_64_call_reg(cd, REG_ITMP1);
4018
4019                 *(jmpInstrPatchPos) = cd->mcodeptr - jmpInstrPos - 1; /*=opcode jmp_imm size*/
4020
4021                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
4022                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
4023                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
4024                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
4025                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
4026                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
4027
4028                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
4029                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
4030                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
4031                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
4032                 x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]);
4033                 x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]);
4034                 x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]);
4035                 x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]);
4036
4037                 x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4038         }
4039 #endif
4040
4041         /* save argument registers on stack -- if we have to */
4042
4043         if ((((m->flags & ACC_STATIC) && iargs > (INT_ARG_CNT - 2)) || iargs > (INT_ARG_CNT - 1)) ||
4044                 (fargs > FLT_ARG_CNT)) {
4045                 s4 paramshiftcnt;
4046                 s4 stackparamcnt;
4047
4048                 paramshiftcnt = 0;
4049                 stackparamcnt = 0;
4050
4051                 /* do we need to shift integer argument register onto stack? */
4052
4053                 if ((m->flags & ACC_STATIC) && iargs > (INT_ARG_CNT - 2)) {
4054                         /* do we need to shift 2 arguments? */
4055                         if (iargs > (INT_ARG_CNT - 1)) {
4056                                 paramshiftcnt = 2;
4057
4058                         } else {
4059                                 paramshiftcnt = 1;
4060                         }
4061
4062                 } else if (iargs > (INT_ARG_CNT - 1)) {
4063                         paramshiftcnt = 1;
4064                 }
4065
4066                 /* calculate required stack space */
4067
4068                 stackparamcnt += (iargs > INT_ARG_CNT) ? iargs - INT_ARG_CNT : 0;
4069                 stackparamcnt += (fargs > FLT_ARG_CNT) ? fargs - FLT_ARG_CNT : 0;
4070
4071                 stackframesize = stackparamcnt + paramshiftcnt;
4072
4073                 /* keep stack 16-byte aligned */
4074                 if (!(stackframesize & 0x1))
4075                         stackframesize++;
4076
4077                 x86_64_alu_imm_reg(cd, X86_64_SUB, stackframesize * 8, REG_SP);
4078
4079                 /* shift integer arguments if required */
4080
4081                 if ((m->flags & ACC_STATIC) && iargs > (INT_ARG_CNT - 2)) {
4082                         /* do we need to shift 2 arguments? */
4083                         if (iargs > (INT_ARG_CNT - 1))
4084                                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 1 * 8);
4085
4086                         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 0 * 8);
4087
4088                 } else if (iargs > (INT_ARG_CNT - 1)) {
4089                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 0 * 8);
4090                 }
4091
4092                 /* copy stack arguments into new stack frame -- if any */
4093                 for (i = 0; i < stackparamcnt; i++) {
4094                         x86_64_mov_membase_reg(cd, REG_SP, (stackframesize + 1 + i) * 8, REG_ITMP1);
4095                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, (paramshiftcnt + i) * 8);
4096                 }
4097
4098         } else {
4099                 /* keep stack 16-byte aligned */
4100                 x86_64_alu_imm_reg(cd, X86_64_SUB, 1 * 8, REG_SP);
4101                 stackframesize = 1;
4102         }
4103
4104         /* shift integer arguments for `env' and `class' arguments */
4105
4106         if (m->flags & ACC_STATIC) {
4107                 /* shift iargs count, if less than INT_ARG_CNT, or all */
4108                 for (i = (iargs < (INT_ARG_CNT - 2)) ? iargs : (INT_ARG_CNT - 2); i >= 0; i--) {
4109                         x86_64_mov_reg_reg(cd, rd->argintregs[i], rd->argintregs[i + 2]);
4110                 }
4111
4112                 /* put class into second argument register */
4113                 x86_64_mov_imm_reg(cd, (u8) m->class, rd->argintregs[1]);
4114
4115         } else {
4116                 /* shift iargs count, if less than INT_ARG_CNT, or all */
4117                 for (i = (iargs < (INT_ARG_CNT - 1)) ? iargs : (INT_ARG_CNT - 1); i >= 0; i--) {
4118                         x86_64_mov_reg_reg(cd, rd->argintregs[i], rd->argintregs[i + 1]);
4119                 }
4120         }
4121
4122         /* put env into first argument register */
4123         x86_64_mov_imm_reg(cd, (u8) &env, rd->argintregs[0]);
4124
4125         /* do the native function call */
4126         x86_64_mov_imm_reg(cd, (u8) f, REG_ITMP1);
4127 #if !defined(STATIC_CLASSPATH)
4128         if (f == NULL)
4129                 (*callAddrPatchPos) = cd->mcodeptr - 8;
4130 #endif
4131         x86_64_call_reg(cd, REG_ITMP1);
4132
4133         /* remove stackframe if there is one */
4134         if (stackframesize) {
4135                 x86_64_alu_imm_reg(cd, X86_64_ADD, stackframesize * 8, REG_SP);
4136         }
4137
4138         if (runverbose) {
4139                 x86_64_alu_imm_reg(cd, X86_64_SUB, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
4140
4141                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
4142                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
4143
4144                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
4145                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
4146                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4147                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4148
4149                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
4150                 x86_64_call_reg(cd, REG_ITMP1);
4151
4152                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
4153                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
4154
4155                 x86_64_alu_imm_reg(cd, X86_64_ADD, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
4156         }
4157
4158         /* check for exception */
4159
4160 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4161         x86_64_push_reg(cd, REG_RESULT);
4162 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4163         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4164         x86_64_call_reg(cd, REG_ITMP3);
4165         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
4166         x86_64_pop_reg(cd, REG_RESULT);
4167 #else
4168         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
4169         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
4170 #endif
4171         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
4172         x86_64_jcc(cd, X86_64_CC_NE, 1);
4173
4174         x86_64_ret(cd);
4175
4176         /* handle exception */
4177
4178 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4179         x86_64_push_reg(cd, REG_ITMP3);
4180 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4181         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4182         x86_64_call_reg(cd, REG_ITMP3);
4183         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4184         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
4185 #else
4186         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4187         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
4188         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4189         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4190 #endif
4191
4192         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
4193         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
4194
4195         x86_64_mov_imm_reg(cd, (u8) asm_handle_nat_exception, REG_ITMP3);
4196         x86_64_jmp_reg(cd, REG_ITMP3);
4197
4198
4199         /* patch in a clinit call if required *************************************/
4200
4201         {
4202                 u1          *xcodeptr;
4203                 clinitref   *cref;
4204                 codegendata *tmpcd;
4205                 u1           xmcode;
4206                 u4           mcode;
4207
4208                 tmpcd = DNEW(codegendata);
4209
4210                 /* there can only be one clinit ref entry                             */
4211                 cref = cd->clinitrefs;
4212
4213                 if (cref) {
4214                         /* Get machine code which is patched back in later. A             */
4215                         /* `call rel32' is 5 bytes long.                                  */
4216                         xcodeptr = cd->mcodebase + cref->branchpos;
4217                         xmcode = *xcodeptr;
4218                         mcode = *((u4 *) (xcodeptr + 1));
4219
4220                         /* patch in `call rel32' to call the following code               */
4221                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4222                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4223
4224                         /* Save current stack pointer into a temporary register.          */
4225                         x86_64_mov_reg_reg(cd, REG_SP, REG_ITMP1);
4226
4227                         /* Push machine code bytes to patch onto the stack.               */
4228                         x86_64_push_imm(cd, (u1) xmcode);
4229                         x86_64_push_imm(cd, (u4) mcode);
4230
4231                         x86_64_push_imm(cd, (u8) cref->class);
4232
4233                         /* Push previously saved stack pointer onto stack.                */
4234                         x86_64_push_reg(cd, REG_ITMP1);
4235
4236                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP1);
4237                         x86_64_jmp_reg(cd, REG_ITMP1);
4238                 }
4239         }
4240
4241         /* Check if the stub size is big enough to hold the whole stub generated. */
4242         /* If not, this can lead into unpredictable crashes, because of heap      */
4243         /* corruption.                                                            */
4244         if ((s4) (cd->mcodeptr - s) > NATIVESTUBSIZE) {
4245                 throw_cacao_exception_exit(string_java_lang_InternalError,
4246                                                                    "Native stub size %d is to small for current stub size %d",
4247                                                                    NATIVESTUBSIZE, (s4) (cd->mcodeptr - s));
4248         }
4249
4250 #if defined(STATISTICS)
4251         if (opt_stat)
4252                 count_nstub_len += NATIVESTUBSIZE;
4253 #endif
4254
4255         /* release dump area */
4256
4257         dump_release(dumpsize);
4258
4259         return s;
4260 }
4261
4262
4263 /* function: removenativestub **************************************************
4264
4265     removes a previously created native-stub from memory
4266
4267 *******************************************************************************/
4268
4269 void removenativestub(u1 *stub)
4270 {
4271         CFREE(stub, NATIVESTUBSIZE);
4272 }
4273
4274
4275 /*
4276  * These are local overrides for various environment variables in Emacs.
4277  * Please do not remove this and leave it at the end of the file, where
4278  * Emacs will automagically detect them.
4279  * ---------------------------------------------------------------------
4280  * Local variables:
4281  * mode: c
4282  * indent-tabs-mode: t
4283  * c-basic-offset: 4
4284  * tab-width: 4
4285  * End:
4286  */