src/vm/jit/x86_64/codegen.c

   1 /* jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003
   4    Institut f. Computersprachen, TU Wien
   5    R. Grafl, A. Krall, C. Kruegel, C. Oates, R. Obermaisser, M. Probst,
   6    S. Ring, E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich,
   7    J. Wenninger
   8
   9    This file is part of CACAO.
  10
  11    This program is free software; you can redistribute it and/or
  12    modify it under the terms of the GNU General Public License as
  13    published by the Free Software Foundation; either version 2, or (at
  14    your option) any later version.
  15
  16    This program is distributed in the hope that it will be useful, but
  17    WITHOUT ANY WARRANTY; without even the implied warranty of
  18    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19    General Public License for more details.
  20
  21    You should have received a copy of the GNU General Public License
  22    along with this program; if not, write to the Free Software
  23    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  24    02111-1307, USA.
  25
  26    Contact: cacao@complang.tuwien.ac.at
  27
  28    Authors: Andreas Krall
  29             Christian Thalinger
  30
  31    $Id: codegen.c 1526 2004-11-17 15:50:27Z twisti $
  32
  33 */
  34
  35 #define _GNU_SOURCE
  36
  37 #include "global.h"
  38 #include <stdio.h>
  39 #include <signal.h>
  40 #include <sys/ucontext.h>
  41 #include "builtin.h"
  42 #include "asmpart.h"
  43 #include "jni.h"
  44 #include "loader.h"
  45 #include "tables.h"
  46 #include "native.h"
  47 #include "jit/jit.h"
  48 #include "jit/reg.h"
  49 #include "jit/parse.h"
  50 #include "jit/x86_64/arch.h"
  51 #include "jit/x86_64/codegen.h"
  52 #include "jit/x86_64/emitfuncs.h"
  53 #include "jit/x86_64/types.h"
  54
  55
  56 /* register descripton - array ************************************************/
  57
  58 /* #define REG_RES   0         reserved register for OS or code generator     */
  59 /* #define REG_RET   1         return value register                          */
  60 /* #define REG_EXC   2         exception value register (only old jit)        */
  61 /* #define REG_SAV   3         (callee) saved register                        */
  62 /* #define REG_TMP   4         scratch temporary register (caller saved)      */
  63 /* #define REG_ARG   5         argument register (caller saved)               */
  64
  65 /* #define REG_END   -1        last entry in tables                           */
  66
  67 int nregdescint[] = {
  68     REG_RET, REG_ARG, REG_ARG, REG_TMP, REG_RES, REG_SAV, REG_ARG, REG_ARG,
  69     REG_ARG, REG_ARG, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV,
  70     REG_END
  71 };
  72
  73
  74 int nregdescfloat[] = {
  75         /*      REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP, */
  76         /*      REG_RES, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV, REG_SAV, */
  77     REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  78     REG_RES, REG_RES, REG_RES, REG_TMP, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  79     REG_END
  80 };
  81
  82
  83 /*******************************************************************************
  84
  85     include independent code generation stuff -- include after register
  86     descriptions to avoid extern definitions
  87
  88 *******************************************************************************/
  89
  90 #include "jit/codegen.inc"
  91 #include "jit/reg.inc"
  92
  93
  94 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
  95 void thread_restartcriticalsection(ucontext_t *uc)
  96 {
  97         void *critical;
  98
  99         critical = thread_checkcritical((void *) uc->uc_mcontext.gregs[REG_RIP]);
 100
 101         if (critical)
 102                 uc->uc_mcontext.gregs[REG_RIP] = (u8) critical;
 103 }
 104 #endif
 105
 106
 107 /* NullPointerException signal handler for hardware null pointer check */
 108
 109 void catch_NullPointerException(int sig, siginfo_t *siginfo, void *_p)
 110 {
 111         sigset_t nsig;
 112         /*      int      instr; */
 113         /*      long     faultaddr; */
 114
 115         struct ucontext *_uc = (struct ucontext *) _p;
 116         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 117         struct sigaction act;
 118         java_objectheader *xptr;
 119
 120         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 121
 122
 123 /*      instr = *((int*)(sigctx->rip)); */
 124 /*      faultaddr = sigctx->sc_regs[(instr >> 16) & 0x1f]; */
 125
 126 /*      if (faultaddr == 0) { */
 127         act.sa_sigaction = (functionptr) catch_NullPointerException; /* reinstall handler */
 128         act.sa_flags = SA_SIGINFO;
 129         sigaction(sig, &act, NULL);
 130
 131         sigemptyset(&nsig);
 132         sigaddset(&nsig, sig);
 133         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 134
 135         xptr = new_nullpointerexception();
 136
 137         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 138         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 139         sigctx->rip = (u8) asm_handle_exception;
 140
 141         return;
 142
 143 /*      } else { */
 144 /*              faultaddr += (long) ((instr << 16) >> 16); */
 145 /*              fprintf(stderr, "faulting address: 0x%08x\n", faultaddr); */
 146 /*              panic("Stack overflow"); */
 147 /*      } */
 148 }
 149
 150
 151 /* ArithmeticException signal handler for hardware divide by zero check */
 152
 153 void catch_ArithmeticException(int sig, siginfo_t *siginfo, void *_p)
 154 {
 155         sigset_t nsig;
 156
 157         struct ucontext *_uc = (struct ucontext *) _p;
 158         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 159         struct sigaction act;
 160         java_objectheader *xptr;
 161
 162         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 163
 164         act.sa_sigaction = (functionptr) catch_ArithmeticException; /* reinstall handler */
 165         act.sa_flags = SA_SIGINFO;
 166         sigaction(sig, &act, NULL);
 167
 168         sigemptyset(&nsig);
 169         sigaddset(&nsig, sig);
 170         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 171
 172         xptr = new_arithmeticexception();
 173
 174         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 175         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 176         sigctx->rip = (u8) asm_handle_exception;
 177
 178         return;
 179 }
 180
 181
 182 void init_exceptions(void)
 183 {
 184         struct sigaction act;
 185
 186         /* install signal handlers we need to convert to exceptions */
 187         sigemptyset(&act.sa_mask);
 188
 189         if (!checknull) {
 190 #if defined(SIGSEGV)
 191                 act.sa_sigaction = (functionptr) catch_NullPointerException;
 192                 act.sa_flags = SA_SIGINFO;
 193                 sigaction(SIGSEGV, &act, NULL);
 194 #endif
 195
 196 #if defined(SIGBUS)
 197                 act.sa_sigaction = (functionptr) catch_NullPointerException;
 198                 act.sa_flags = SA_SIGINFO;
 199                 sigaction(SIGBUS, &act, NULL);
 200 #endif
 201         }
 202
 203         act.sa_sigaction = (functionptr) catch_ArithmeticException;
 204         act.sa_flags = SA_SIGINFO;
 205         sigaction(SIGFPE, &act, NULL);
 206 }
 207
 208
 209 /* function gen_mcode **********************************************************
 210
 211         generates machine code
 212
 213 *******************************************************************************/
 214
 215 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
 216 {
 217         s4 len, s1, s2, s3, d;
 218         s8 a;
 219         s4 parentargs_base;
 220         stackptr        src;
 221         varinfo        *var;
 222         basicblock     *bptr;
 223         instruction    *iptr;
 224         exceptiontable *ex;
 225
 226         {
 227         s4 i, p, pa, t, l;
 228         s4 savedregs_num;
 229
 230         savedregs_num = 0;
 231
 232         /* space to save used callee saved registers */
 233
 234         savedregs_num += (rd->savintregcnt - rd->maxsavintreguse);
 235         savedregs_num += (rd->savfltregcnt - rd->maxsavfltreguse);
 236
 237         parentargs_base = rd->maxmemuse + savedregs_num;
 238
 239 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 240
 241         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 242                 parentargs_base++;
 243
 244 #endif
 245
 246     /* keep stack 16-byte aligned for calls into native code e.g. libc or jni */
 247     /* (alignment problems with movaps)                                       */
 248
 249         if (!(parentargs_base & 0x1)) {
 250                 parentargs_base++;
 251         }
 252
 253         /* create method header */
 254
 255         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 256         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 257
 258 #if defined(USE_THREADS)
 259
 260         /* IsSync contains the offset relative to the stack pointer for the
 261            argument of monitor_exit used in the exception handler. Since the
 262            offset could be zero and give a wrong meaning of the flag it is
 263            offset by one.
 264         */
 265
 266         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 267                 (void) dseg_adds4(cd, (rd->maxmemuse + 1) * 8);     /* IsSync         */
 268         else
 269
 270 #endif
 271
 272                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 273
 274         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 275         (void) dseg_adds4(cd, rd->savintregcnt - rd->maxsavintreguse);/* IntSave  */
 276         (void) dseg_adds4(cd, rd->savfltregcnt - rd->maxsavfltreguse);/* FltSave  */
 277         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 278
 279         /* create exception table */
 280
 281         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 282                 dseg_addtarget(cd, ex->start);
 283                 dseg_addtarget(cd, ex->end);
 284                 dseg_addtarget(cd, ex->handler);
 285                 (void) dseg_addaddress(cd, ex->catchtype);
 286         }
 287
 288         /* initialize mcode variables */
 289
 290         cd->mcodeptr = (u1 *) cd->mcodebase;
 291         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 292         MCODECHECK(128 + m->paramcount);
 293
 294         /* create stack frame (if necessary) */
 295
 296         if (parentargs_base) {
 297                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 298         }
 299
 300         /* save return address and used callee saved registers */
 301
 302         p = parentargs_base;
 303         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
 304                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 305         }
 306         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
 307                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 308         }
 309
 310         /* save monitorenter argument */
 311
 312 #if defined(USE_THREADS)
 313         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 314                 if (m->flags & ACC_STATIC) {
 315                         x86_64_mov_imm_reg(cd, (s8) m->class, REG_ITMP1);
 316                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, rd->maxmemuse * 8);
 317
 318                 } else {
 319                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, rd->maxmemuse * 8);
 320                 }
 321         }
 322 #endif
 323
 324         /* copy argument registers to stack and call trace function with pointer
 325            to arguments on stack.
 326         */
 327         if (runverbose) {
 328                 x86_64_alu_imm_reg(cd, X86_64_SUB, (6 + 8 + 1 + 1) * 8, REG_SP);
 329
 330                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
 331                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
 332                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
 333                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
 334                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
 335                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
 336
 337                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
 338                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
 339                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
 340                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
 341 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8); */
 342 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8); */
 343 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8); */
 344 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8); */
 345
 346                 for (p = 0, l = 0; p < m->paramcount; p++) {
 347                         t = m->paramtypes[p];
 348
 349                         if (IS_FLT_DBL_TYPE(t)) {
 350                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
 351                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
 352                                 }
 353
 354                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 355                                 l++;
 356                         }
 357                 }
 358
 359                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP2);
 360                 x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 361                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
 362                 x86_64_call_reg(cd, REG_ITMP1);
 363
 364                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
 365                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
 366                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
 367                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
 368                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
 369                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
 370
 371                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
 372                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
 373                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
 374                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
 375 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]); */
 376 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]); */
 377 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]); */
 378 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]); */
 379
 380                 x86_64_alu_imm_reg(cd, X86_64_ADD, (6 + 8 + 1 + 1) * 8, REG_SP);
 381         }
 382
 383         /* take arguments out of register or stack frame */
 384
 385         for (p = 0, l = 0, s1 = 0, s2 = 0; p < m->paramcount; p++) {
 386                 t = m->paramtypes[p];
 387                 var = &(rd->locals[l][t]);
 388                 l++;
 389                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 390                         l++;
 391                 if (var->type < 0) {
 392                         if (IS_INT_LNG_TYPE(t)) {
 393                                 s1++;
 394                         } else {
 395                                 s2++;
 396                         }
 397                         continue;
 398                 }
 399                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 400                         if (s1 < INT_ARG_CNT) {                  /* register arguments    */
 401                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 402                                         M_INTMOVE(rd->argintregs[s1], var->regoff);
 403
 404                                 } else {                             /* reg arg -> spilled    */
 405                                     x86_64_mov_reg_membase(cd, rd->argintregs[s1], REG_SP, var->regoff * 8);
 406                                 }
 407
 408                         } else {                                 /* stack arguments       */
 409                                 pa = s1 - INT_ARG_CNT;
 410                                 if (s2 >= FLT_ARG_CNT) {
 411                                         pa += s2 - FLT_ARG_CNT;
 412                                 }
 413                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 414                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);    /* + 8 for return address */
 415                                 } else {                             /* stack arg -> spilled  */
 416                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_ITMP1);    /* + 8 for return address */
 417                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, var->regoff * 8);
 418                                 }
 419                         }
 420                         s1++;
 421
 422                 } else {                                     /* floating args         */
 423                         if (s2 < FLT_ARG_CNT) {                /* register arguments    */
 424                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 425                                         M_FLTMOVE(rd->argfltregs[s2], var->regoff);
 426
 427                                 } else {                                         /* reg arg -> spilled    */
 428                                         x86_64_movq_reg_membase(cd, rd->argfltregs[s2], REG_SP, var->regoff * 8);
 429                                 }
 430
 431                         } else {                                 /* stack arguments       */
 432                                 pa = s2 - FLT_ARG_CNT;
 433                                 if (s1 >= INT_ARG_CNT) {
 434                                         pa += s1 - INT_ARG_CNT;
 435                                 }
 436                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 437                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);
 438
 439                                 } else {
 440                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_FTMP1);
 441                                         x86_64_movq_reg_membase(cd, REG_FTMP1, REG_SP, var->regoff * 8);
 442                                 }
 443                         }
 444                         s2++;
 445                 }
 446         }  /* end for */
 447
 448         /* call monitorenter function */
 449
 450 #if defined(USE_THREADS)
 451         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 452                 s8 func_enter = (m->flags & ACC_STATIC) ?
 453                         (s8) builtin_staticmonitorenter : (s8) builtin_monitorenter;
 454                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
 455                 x86_64_mov_imm_reg(cd, func_enter, REG_ITMP1);
 456                 x86_64_call_reg(cd, REG_ITMP1);
 457         }
 458 #endif
 459         }
 460
 461         /* end of header generation */
 462
 463         /* walk through all basic blocks */
 464         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 465
 466                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 467
 468                 if (bptr->flags >= BBREACHED) {
 469
 470                         /* branch resolving */
 471
 472                         branchref *bref;
 473                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 474                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 475                                                                   bref->branchpos,
 476                                                                   bptr->mpc);
 477                         }
 478
 479                 /* copy interface registers to their destination */
 480
 481                 src = bptr->instack;
 482                 len = bptr->indepth;
 483                 MCODECHECK(64 + len);
 484                 while (src != NULL) {
 485                         len--;
 486                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 487                                 if (bptr->type == BBTYPE_SBR) {
 488                                         d = reg_of_var(rd, src, REG_ITMP1);
 489                                         x86_64_pop_reg(cd, d);
 490                                         store_reg_to_var_int(src, d);
 491
 492                                 } else if (bptr->type == BBTYPE_EXH) {
 493                                         d = reg_of_var(rd, src, REG_ITMP1);
 494                                         M_INTMOVE(REG_ITMP1, d);
 495                                         store_reg_to_var_int(src, d);
 496                                 }
 497
 498                         } else {
 499                                 d = reg_of_var(rd, src, REG_ITMP1);
 500                                 if ((src->varkind != STACKVAR)) {
 501                                         s2 = src->type;
 502                                         if (IS_FLT_DBL_TYPE(s2)) {
 503                                                 s1 = rd->interfaces[len][s2].regoff;
 504                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 505                                                         M_FLTMOVE(s1, d);
 506
 507                                                 } else {
 508                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 509                                                 }
 510                                                 store_reg_to_var_flt(src, d);
 511
 512                                         } else {
 513                                                 s1 = rd->interfaces[len][s2].regoff;
 514                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 515                                                         M_INTMOVE(s1, d);
 516
 517                                                 } else {
 518                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 519                                                 }
 520                                                 store_reg_to_var_int(src, d);
 521                                         }
 522                                 }
 523                         }
 524                         src = src->prev;
 525                 }
 526
 527                 /* walk through all instructions */
 528
 529                 src = bptr->instack;
 530                 len = bptr->icount;
 531                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 532
 533                         MCODECHECK(64);   /* an instruction usually needs < 64 words      */
 534                         switch (iptr->opc) {
 535
 536                         case ICMD_NOP:    /* ...  ==> ...                                 */
 537                                 break;
 538
 539                         case ICMD_NULLCHECKPOP: /* ..., objectref  ==> ...                */
 540                                 if (src->flags & INMEMORY) {
 541                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 542
 543                                 } else {
 544                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 545                                 }
 546                                 x86_64_jcc(cd, X86_64_CC_E, 0);
 547                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 548                                 break;
 549
 550                 /* constant operations ************************************************/
 551
 552                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 553                                       /* op1 = 0, val.i = constant                    */
 554
 555                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 556                         if (iptr->val.i == 0) {
 557                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 558                         } else {
 559                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 560                         }
 561                         store_reg_to_var_int(iptr->dst, d);
 562                         break;
 563
 564                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 565                                       /* op1 = 0, val.a = constant                    */
 566
 567                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 568                         if (iptr->val.a == 0) {
 569                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 570                         } else {
 571                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 572                         }
 573                         store_reg_to_var_int(iptr->dst, d);
 574                         break;
 575
 576                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 577                                       /* op1 = 0, val.l = constant                    */
 578
 579                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 580                         if (iptr->val.l == 0) {
 581                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 582                         } else {
 583                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 584                         }
 585                         store_reg_to_var_int(iptr->dst, d);
 586                         break;
 587
 588                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 589                                       /* op1 = 0, val.f = constant                    */
 590
 591                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 592                         a = dseg_addfloat(cd, iptr->val.f);
 593                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 594                         store_reg_to_var_flt(iptr->dst, d);
 595                         break;
 596
 597                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 598                                       /* op1 = 0, val.d = constant                    */
 599
 600                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 601                         a = dseg_adddouble(cd, iptr->val.d);
 602                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 603                         store_reg_to_var_flt(iptr->dst, d);
 604                         break;
 605
 606
 607                 /* load/store operations **********************************************/
 608
 609                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 610                                       /* op1 = local variable                         */
 611
 612                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 613                         if ((iptr->dst->varkind == LOCALVAR) &&
 614                             (iptr->dst->varnum == iptr->op1)) {
 615                                 break;
 616                         }
 617                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 618                         if (var->flags & INMEMORY) {
 619                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 620                                 store_reg_to_var_int(iptr->dst, d);
 621
 622                         } else {
 623                                 if (iptr->dst->flags & INMEMORY) {
 624                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 625
 626                                 } else {
 627                                         M_INTMOVE(var->regoff, d);
 628                                 }
 629                         }
 630                         break;
 631
 632                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 633                 case ICMD_ALOAD:      /* op1 = local variable                         */
 634
 635                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 636                         if ((iptr->dst->varkind == LOCALVAR) &&
 637                             (iptr->dst->varnum == iptr->op1)) {
 638                                 break;
 639                         }
 640                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 641                         if (var->flags & INMEMORY) {
 642                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 643                                 store_reg_to_var_int(iptr->dst, d);
 644
 645                         } else {
 646                                 if (iptr->dst->flags & INMEMORY) {
 647                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 648
 649                                 } else {
 650                                         M_INTMOVE(var->regoff, d);
 651                                 }
 652                         }
 653                         break;
 654
 655                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 656                 case ICMD_DLOAD:      /* op1 = local variable                         */
 657
 658                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 659                         if ((iptr->dst->varkind == LOCALVAR) &&
 660                             (iptr->dst->varnum == iptr->op1)) {
 661                                 break;
 662                         }
 663                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 664                         if (var->flags & INMEMORY) {
 665                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 666                                 store_reg_to_var_flt(iptr->dst, d);
 667
 668                         } else {
 669                                 if (iptr->dst->flags & INMEMORY) {
 670                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 671
 672                                 } else {
 673                                         M_FLTMOVE(var->regoff, d);
 674                                 }
 675                         }
 676                         break;
 677
 678                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 679                 case ICMD_LSTORE:     /* op1 = local variable                         */
 680                 case ICMD_ASTORE:
 681
 682                         if ((src->varkind == LOCALVAR) &&
 683                             (src->varnum == iptr->op1)) {
 684                                 break;
 685                         }
 686                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 687                         if (var->flags & INMEMORY) {
 688                                 var_to_reg_int(s1, src, REG_ITMP1);
 689                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 690
 691                         } else {
 692                                 var_to_reg_int(s1, src, var->regoff);
 693                                 M_INTMOVE(s1, var->regoff);
 694                         }
 695                         break;
 696
 697                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 698                 case ICMD_DSTORE:     /* op1 = local variable                         */
 699
 700                         if ((src->varkind == LOCALVAR) &&
 701                             (src->varnum == iptr->op1)) {
 702                                 break;
 703                         }
 704                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 705                         if (var->flags & INMEMORY) {
 706                                 var_to_reg_flt(s1, src, REG_FTMP1);
 707                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 708
 709                         } else {
 710                                 var_to_reg_flt(s1, src, var->regoff);
 711                                 M_FLTMOVE(s1, var->regoff);
 712                         }
 713                         break;
 714
 715
 716                 /* pop/dup/swap operations ********************************************/
 717
 718                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 719
 720                 case ICMD_POP:        /* ..., value  ==> ...                          */
 721                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 722                         break;
 723
 724                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 725                         M_COPY(src, iptr->dst);
 726                         break;
 727
 728                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 729
 730                         M_COPY(src,       iptr->dst);
 731                         M_COPY(src->prev, iptr->dst->prev);
 732                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 733                         break;
 734
 735                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 736
 737                         M_COPY(src,             iptr->dst);
 738                         M_COPY(src->prev,       iptr->dst->prev);
 739                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 740                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 741                         break;
 742
 743                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 744
 745                         M_COPY(src,       iptr->dst);
 746                         M_COPY(src->prev, iptr->dst->prev);
 747                         break;
 748
 749                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 750
 751                         M_COPY(src,             iptr->dst);
 752                         M_COPY(src->prev,       iptr->dst->prev);
 753                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 754                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 755                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 756                         break;
 757
 758                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 759
 760                         M_COPY(src,                   iptr->dst);
 761                         M_COPY(src->prev,             iptr->dst->prev);
 762                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 763                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 764                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 765                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 766                         break;
 767
 768                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 769
 770                         M_COPY(src,       iptr->dst->prev);
 771                         M_COPY(src->prev, iptr->dst);
 772                         break;
 773
 774
 775                 /* integer operations *************************************************/
 776
 777                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 778
 779                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 780                         if (iptr->dst->flags & INMEMORY) {
 781                                 if (src->flags & INMEMORY) {
 782                                         if (src->regoff == iptr->dst->regoff) {
 783                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 784
 785                                         } else {
 786                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 787                                                 x86_64_negl_reg(cd, REG_ITMP1);
 788                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 789                                         }
 790
 791                                 } else {
 792                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 793                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 794                                 }
 795
 796                         } else {
 797                                 if (src->flags & INMEMORY) {
 798                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 799                                         x86_64_negl_reg(cd, d);
 800
 801                                 } else {
 802                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 803                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 804                                 }
 805                         }
 806                         break;
 807
 808                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 809
 810                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 811                         if (iptr->dst->flags & INMEMORY) {
 812                                 if (src->flags & INMEMORY) {
 813                                         if (src->regoff == iptr->dst->regoff) {
 814                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 815
 816                                         } else {
 817                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 818                                                 x86_64_neg_reg(cd, REG_ITMP1);
 819                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 820                                         }
 821
 822                                 } else {
 823                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 824                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 825                                 }
 826
 827                         } else {
 828                                 if (src->flags & INMEMORY) {
 829                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 830                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 831
 832                                 } else {
 833                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 834                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 835                                 }
 836                         }
 837                         break;
 838
 839                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 840
 841                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 842                         if (src->flags & INMEMORY) {
 843                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 844
 845                         } else {
 846                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 847                         }
 848                         store_reg_to_var_int(iptr->dst, d);
 849                         break;
 850
 851                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 852
 853                         var_to_reg_int(s1, src, REG_ITMP1);
 854                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 855                         M_INTMOVE(s1, d);
 856                         store_reg_to_var_int(iptr->dst, d);
 857                         break;
 858
 859                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 860
 861                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 862                         if (src->flags & INMEMORY) {
 863                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 864
 865                         } else {
 866                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 867                         }
 868                         store_reg_to_var_int(iptr->dst, d);
 869                         break;
 870
 871                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 872
 873                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 874                         if (src->flags & INMEMORY) {
 875                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 876
 877                         } else {
 878                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 879                         }
 880                         store_reg_to_var_int(iptr->dst, d);
 881                         break;
 882
 883                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 884
 885                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 886                         if (src->flags & INMEMORY) {
 887                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 888
 889                         } else {
 890                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 891                         }
 892                         store_reg_to_var_int(iptr->dst, d);
 893                         break;
 894
 895
 896                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 897
 898                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 899                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 900                         break;
 901
 902                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 903                                       /* val.i = constant                             */
 904
 905                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 906                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 907                         break;
 908
 909                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 910
 911                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 912                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 913                         break;
 914
 915                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 916                                       /* val.l = constant                             */
 917
 918                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 919                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 920                         break;
 921
 922                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 923
 924                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 925                         if (iptr->dst->flags & INMEMORY) {
 926                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 927                                         if (src->prev->regoff == iptr->dst->regoff) {
 928                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 929                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 930
 931                                         } else {
 932                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 933                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 934                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 935                                         }
 936
 937                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 938                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 939                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 940                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 941
 942                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 943                                         if (src->prev->regoff == iptr->dst->regoff) {
 944                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 945
 946                                         } else {
 947                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 948                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 949                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 950                                         }
 951
 952                                 } else {
 953                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 954                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 955                                 }
 956
 957                         } else {
 958                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 959                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 960                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 961
 962                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 963                                         M_INTMOVE(src->prev->regoff, d);
 964                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 965
 966                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 967                                         /* workaround for reg alloc */
 968                                         if (src->regoff == iptr->dst->regoff) {
 969                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 970                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 971                                                 M_INTMOVE(REG_ITMP1, d);
 972
 973                                         } else {
 974                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 975                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 976                                         }
 977
 978                                 } else {
 979                                         /* workaround for reg alloc */
 980                                         if (src->regoff == iptr->dst->regoff) {
 981                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 982                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 983                                                 M_INTMOVE(REG_ITMP1, d);
 984
 985                                         } else {
 986                                                 M_INTMOVE(src->prev->regoff, d);
 987                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 988                                         }
 989                                 }
 990                         }
 991                         break;
 992
 993                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 994                                       /* val.i = constant                             */
 995
 996                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 997                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 998                         break;
 999
1000                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1001
1002                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1003                         if (iptr->dst->flags & INMEMORY) {
1004                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1005                                         if (src->prev->regoff == iptr->dst->regoff) {
1006                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1007                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1008
1009                                         } else {
1010                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1011                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1012                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1013                                         }
1014
1015                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1016                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1017                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1018                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1019
1020                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1021                                         if (src->prev->regoff == iptr->dst->regoff) {
1022                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1023
1024                                         } else {
1025                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1026                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1027                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1028                                         }
1029
1030                                 } else {
1031                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1032                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1033                                 }
1034
1035                         } else {
1036                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1037                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1038                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1039
1040                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1041                                         M_INTMOVE(src->prev->regoff, d);
1042                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1043
1044                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1045                                         /* workaround for reg alloc */
1046                                         if (src->regoff == iptr->dst->regoff) {
1047                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1048                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1049                                                 M_INTMOVE(REG_ITMP1, d);
1050
1051                                         } else {
1052                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1053                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1054                                         }
1055
1056                                 } else {
1057                                         /* workaround for reg alloc */
1058                                         if (src->regoff == iptr->dst->regoff) {
1059                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1060                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1061                                                 M_INTMOVE(REG_ITMP1, d);
1062
1063                                         } else {
1064                                                 M_INTMOVE(src->prev->regoff, d);
1065                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1066                                         }
1067                                 }
1068                         }
1069                         break;
1070
1071                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1072                                       /* val.l = constant                             */
1073
1074                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1075                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1076                         break;
1077
1078                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1079
1080                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1081                         if (iptr->dst->flags & INMEMORY) {
1082                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1083                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1084                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1085                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1086
1087                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1088                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1089                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1090                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1091
1092                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1093                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1094                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1095                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1096
1097                                 } else {
1098                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1099                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1100                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1101                                 }
1102
1103                         } else {
1104                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1105                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1106                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1107
1108                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1109                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1110                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1111
1112                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1113                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1114                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1115
1116                                 } else {
1117                                         if (src->regoff == iptr->dst->regoff) {
1118                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1119
1120                                         } else {
1121                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1122                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1123                                         }
1124                                 }
1125                         }
1126                         break;
1127
1128                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1129                                       /* val.i = constant                             */
1130
1131                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1132                         if (iptr->dst->flags & INMEMORY) {
1133                                 if (src->flags & INMEMORY) {
1134                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1135                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1136
1137                                 } else {
1138                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1139                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1140                                 }
1141
1142                         } else {
1143                                 if (src->flags & INMEMORY) {
1144                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1145
1146                                 } else {
1147                                         if (iptr->val.i == 2) {
1148                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1149                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1150
1151                                         } else {
1152                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1153                                         }
1154                                 }
1155                         }
1156                         break;
1157
1158                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1159
1160                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1161                         if (iptr->dst->flags & INMEMORY) {
1162                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1163                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1164                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1165                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1166
1167                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1168                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1169                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1170                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1171
1172                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1173                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1174                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1175                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1176
1177                                 } else {
1178                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1179                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1180                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1181                                 }
1182
1183                         } else {
1184                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1185                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1186                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1187
1188                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1189                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1190                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1191
1192                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1193                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1194                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1195
1196                                 } else {
1197                                         if (src->regoff == iptr->dst->regoff) {
1198                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1199
1200                                         } else {
1201                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1202                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1203                                         }
1204                                 }
1205                         }
1206                         break;
1207
1208                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1209                                       /* val.l = constant                             */
1210
1211                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1212                         if (iptr->dst->flags & INMEMORY) {
1213                                 if (src->flags & INMEMORY) {
1214                                         if (x86_64_is_imm32(iptr->val.l)) {
1215                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1216
1217                                         } else {
1218                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1219                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1220                                         }
1221                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1222
1223                                 } else {
1224                                         if (x86_64_is_imm32(iptr->val.l)) {
1225                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1226
1227                                         } else {
1228                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1229                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1230                                         }
1231                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1232                                 }
1233
1234                         } else {
1235                                 if (src->flags & INMEMORY) {
1236                                         if (x86_64_is_imm32(iptr->val.l)) {
1237                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1238
1239                                         } else {
1240                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1241                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1242                                         }
1243
1244                                 } else {
1245                                         /* should match in many cases */
1246                                         if (iptr->val.l == 2) {
1247                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1248                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1249
1250                                         } else {
1251                                                 if (x86_64_is_imm32(iptr->val.l)) {
1252                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1253
1254                                                 } else {
1255                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1256                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1257                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1258                                                 }
1259                                         }
1260                                 }
1261                         }
1262                         break;
1263
1264                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1265
1266                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1267                 if (src->prev->flags & INMEMORY) {
1268                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1269
1270                         } else {
1271                                 M_INTMOVE(src->prev->regoff, RAX);
1272                         }
1273
1274                         if (src->flags & INMEMORY) {
1275                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1276
1277                         } else {
1278                                 M_INTMOVE(src->regoff, REG_ITMP3);
1279                         }
1280                         gen_div_check(src);
1281
1282                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1283                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1284                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1285                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1286
1287                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1288                         x86_64_cltd(cd);
1289                         x86_64_idivl_reg(cd, REG_ITMP3);
1290
1291                         if (iptr->dst->flags & INMEMORY) {
1292                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1293                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1294
1295                         } else {
1296                                 M_INTMOVE(RAX, iptr->dst->regoff);
1297
1298                                 if (iptr->dst->regoff != RDX) {
1299                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1300                                 }
1301                         }
1302                         break;
1303
1304                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1305
1306                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1307                         if (src->prev->flags & INMEMORY) {
1308                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1309
1310                         } else {
1311                                 M_INTMOVE(src->prev->regoff, RAX);
1312                         }
1313
1314                         if (src->flags & INMEMORY) {
1315                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1316
1317                         } else {
1318                                 M_INTMOVE(src->regoff, REG_ITMP3);
1319                         }
1320                         gen_div_check(src);
1321
1322                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1323                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1324                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1325                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1326                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1327
1328                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1329                         x86_64_cltd(cd);
1330                         x86_64_idivl_reg(cd, REG_ITMP3);
1331
1332                         if (iptr->dst->flags & INMEMORY) {
1333                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1334                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1335
1336                         } else {
1337                                 M_INTMOVE(RDX, iptr->dst->regoff);
1338
1339                                 if (iptr->dst->regoff != RDX) {
1340                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1341                                 }
1342                         }
1343                         break;
1344
1345                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1346                                       /* val.i = constant                             */
1347
1348                         var_to_reg_int(s1, src, REG_ITMP1);
1349                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1350                         M_INTMOVE(s1, REG_ITMP1);
1351                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1352                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1353                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1354                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1355                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1356                         store_reg_to_var_int(iptr->dst, d);
1357                         break;
1358
1359                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1360                                       /* val.i = constant                             */
1361
1362                         var_to_reg_int(s1, src, REG_ITMP1);
1363                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1364                         M_INTMOVE(s1, REG_ITMP1);
1365                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1366                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1367                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1368                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1369                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1370                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1371                         store_reg_to_var_int(iptr->dst, d);
1372                         break;
1373
1374
1375                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1376
1377                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1378                 if (src->prev->flags & INMEMORY) {
1379                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1380
1381                         } else {
1382                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1383                         }
1384
1385                         if (src->flags & INMEMORY) {
1386                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1387
1388                         } else {
1389                                 M_INTMOVE(src->regoff, REG_ITMP3);
1390                         }
1391                         gen_div_check(src);
1392
1393                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1394                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1395                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1396                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1397                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1398
1399                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1400                         x86_64_cqto(cd);
1401                         x86_64_idiv_reg(cd, REG_ITMP3);
1402
1403                         if (iptr->dst->flags & INMEMORY) {
1404                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1405                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1406
1407                         } else {
1408                                 M_INTMOVE(RAX, iptr->dst->regoff);
1409
1410                                 if (iptr->dst->regoff != RDX) {
1411                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1412                                 }
1413                         }
1414                         break;
1415
1416                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1417
1418                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1419                         if (src->prev->flags & INMEMORY) {
1420                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1421
1422                         } else {
1423                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1424                         }
1425
1426                         if (src->flags & INMEMORY) {
1427                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1428
1429                         } else {
1430                                 M_INTMOVE(src->regoff, REG_ITMP3);
1431                         }
1432                         gen_div_check(src);
1433
1434                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1435                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1436                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1437                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1438                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1439                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1440
1441                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1442                         x86_64_cqto(cd);
1443                         x86_64_idiv_reg(cd, REG_ITMP3);
1444
1445                         if (iptr->dst->flags & INMEMORY) {
1446                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1447                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1448
1449                         } else {
1450                                 M_INTMOVE(RDX, iptr->dst->regoff);
1451
1452                                 if (iptr->dst->regoff != RDX) {
1453                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1454                                 }
1455                         }
1456                         break;
1457
1458                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1459                                       /* val.i = constant                             */
1460
1461                         var_to_reg_int(s1, src, REG_ITMP1);
1462                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1463                         M_INTMOVE(s1, REG_ITMP1);
1464                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1465                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1466                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1467                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1468                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1469                         store_reg_to_var_int(iptr->dst, d);
1470                         break;
1471
1472                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1473                                       /* val.l = constant                             */
1474
1475                         var_to_reg_int(s1, src, REG_ITMP1);
1476                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1477                         M_INTMOVE(s1, REG_ITMP1);
1478                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1479                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1480                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1481                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1482                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1483                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1484                         store_reg_to_var_int(iptr->dst, d);
1485                         break;
1486
1487                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1488
1489                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1490                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1491                         break;
1492
1493                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1494                                       /* val.i = constant                             */
1495
1496                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1497                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1498                         break;
1499
1500                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1501
1502                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1503                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1504                         break;
1505
1506                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1507                                       /* val.i = constant                             */
1508
1509                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1510                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1511                         break;
1512
1513                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1514
1515                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1516                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1517                         break;
1518
1519                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1520                                       /* val.i = constant                             */
1521
1522                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1523                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1524                         break;
1525
1526                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1527
1528                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1529                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1530                         break;
1531
1532         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1533                                           /* val.i = constant                             */
1534
1535                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1536                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1537                         break;
1538
1539                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1540
1541                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1542                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1543                         break;
1544
1545                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1546                                       /* val.i = constant                             */
1547
1548                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1549                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1550                         break;
1551
1552                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1553
1554                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1555                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1556                         break;
1557
1558                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1559                                       /* val.l = constant                             */
1560
1561                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1562                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1563                         break;
1564
1565                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1566
1567                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1568                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1569                         break;
1570
1571                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1572                                       /* val.i = constant                             */
1573
1574                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1575                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1576                         break;
1577
1578                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1579
1580                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1581                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1582                         break;
1583
1584                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1585                                       /* val.l = constant                             */
1586
1587                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1588                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1589                         break;
1590
1591                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1592
1593                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1594                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1595                         break;
1596
1597                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1598                                       /* val.i = constant                             */
1599
1600                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1601                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1602                         break;
1603
1604                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1605
1606                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1607                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1608                         break;
1609
1610                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1611                                       /* val.l = constant                             */
1612
1613                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1614                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1615                         break;
1616
1617                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1618
1619                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1620                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1621                         break;
1622
1623                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1624                                       /* val.i = constant                             */
1625
1626                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1627                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1628                         break;
1629
1630                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1631
1632                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1633                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1634                         break;
1635
1636                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1637                                       /* val.l = constant                             */
1638
1639                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1640                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1641                         break;
1642
1643
1644                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1645                                       /* op1 = variable, val.i = constant             */
1646
1647                         /* using inc and dec is definitely faster than add -- tested      */
1648                         /* with sieve                                                     */
1649
1650                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1651                         d = var->regoff;
1652                         if (var->flags & INMEMORY) {
1653                                 if (iptr->val.i == 1) {
1654                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1655
1656                                 } else if (iptr->val.i == -1) {
1657                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1658
1659                                 } else {
1660                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1661                                 }
1662
1663                         } else {
1664                                 if (iptr->val.i == 1) {
1665                                         x86_64_incl_reg(cd, d);
1666
1667                                 } else if (iptr->val.i == -1) {
1668                                         x86_64_decl_reg(cd, d);
1669
1670                                 } else {
1671                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1672                                 }
1673                         }
1674                         break;
1675
1676
1677                 /* floating operations ************************************************/
1678
1679                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1680
1681                         var_to_reg_flt(s1, src, REG_FTMP1);
1682                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1683                         a = dseg_adds4(cd, 0x80000000);
1684                         M_FLTMOVE(s1, d);
1685                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1686                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1687                         store_reg_to_var_flt(iptr->dst, d);
1688                         break;
1689
1690                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1691
1692                         var_to_reg_flt(s1, src, REG_FTMP1);
1693                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1694                         a = dseg_adds8(cd, 0x8000000000000000);
1695                         M_FLTMOVE(s1, d);
1696                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1697                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1698                         store_reg_to_var_flt(iptr->dst, d);
1699                         break;
1700
1701                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1702
1703                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1704                         var_to_reg_flt(s2, src, REG_FTMP2);
1705                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1706                         if (s1 == d) {
1707                                 x86_64_addss_reg_reg(cd, s2, d);
1708                         } else if (s2 == d) {
1709                                 x86_64_addss_reg_reg(cd, s1, d);
1710                         } else {
1711                                 M_FLTMOVE(s1, d);
1712                                 x86_64_addss_reg_reg(cd, s2, d);
1713                         }
1714                         store_reg_to_var_flt(iptr->dst, d);
1715                         break;
1716
1717                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1718
1719                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1720                         var_to_reg_flt(s2, src, REG_FTMP2);
1721                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1722                         if (s1 == d) {
1723                                 x86_64_addsd_reg_reg(cd, s2, d);
1724                         } else if (s2 == d) {
1725                                 x86_64_addsd_reg_reg(cd, s1, d);
1726                         } else {
1727                                 M_FLTMOVE(s1, d);
1728                                 x86_64_addsd_reg_reg(cd, s2, d);
1729                         }
1730                         store_reg_to_var_flt(iptr->dst, d);
1731                         break;
1732
1733                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1734
1735                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1736                         var_to_reg_flt(s2, src, REG_FTMP2);
1737                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1738                         if (s2 == d) {
1739                                 M_FLTMOVE(s2, REG_FTMP2);
1740                                 s2 = REG_FTMP2;
1741                         }
1742                         M_FLTMOVE(s1, d);
1743                         x86_64_subss_reg_reg(cd, s2, d);
1744                         store_reg_to_var_flt(iptr->dst, d);
1745                         break;
1746
1747                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1748
1749                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1750                         var_to_reg_flt(s2, src, REG_FTMP2);
1751                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1752                         if (s2 == d) {
1753                                 M_FLTMOVE(s2, REG_FTMP2);
1754                                 s2 = REG_FTMP2;
1755                         }
1756                         M_FLTMOVE(s1, d);
1757                         x86_64_subsd_reg_reg(cd, s2, d);
1758                         store_reg_to_var_flt(iptr->dst, d);
1759                         break;
1760
1761                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1762
1763                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1764                         var_to_reg_flt(s2, src, REG_FTMP2);
1765                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1766                         if (s1 == d) {
1767                                 x86_64_mulss_reg_reg(cd, s2, d);
1768                         } else if (s2 == d) {
1769                                 x86_64_mulss_reg_reg(cd, s1, d);
1770                         } else {
1771                                 M_FLTMOVE(s1, d);
1772                                 x86_64_mulss_reg_reg(cd, s2, d);
1773                         }
1774                         store_reg_to_var_flt(iptr->dst, d);
1775                         break;
1776
1777                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1778
1779                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1780                         var_to_reg_flt(s2, src, REG_FTMP2);
1781                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1782                         if (s1 == d) {
1783                                 x86_64_mulsd_reg_reg(cd, s2, d);
1784                         } else if (s2 == d) {
1785                                 x86_64_mulsd_reg_reg(cd, s1, d);
1786                         } else {
1787                                 M_FLTMOVE(s1, d);
1788                                 x86_64_mulsd_reg_reg(cd, s2, d);
1789                         }
1790                         store_reg_to_var_flt(iptr->dst, d);
1791                         break;
1792
1793                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1794
1795                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1796                         var_to_reg_flt(s2, src, REG_FTMP2);
1797                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1798                         if (s2 == d) {
1799                                 M_FLTMOVE(s2, REG_FTMP2);
1800                                 s2 = REG_FTMP2;
1801                         }
1802                         M_FLTMOVE(s1, d);
1803                         x86_64_divss_reg_reg(cd, s2, d);
1804                         store_reg_to_var_flt(iptr->dst, d);
1805                         break;
1806
1807                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1808
1809                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1810                         var_to_reg_flt(s2, src, REG_FTMP2);
1811                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1812                         if (s2 == d) {
1813                                 M_FLTMOVE(s2, REG_FTMP2);
1814                                 s2 = REG_FTMP2;
1815                         }
1816                         M_FLTMOVE(s1, d);
1817                         x86_64_divsd_reg_reg(cd, s2, d);
1818                         store_reg_to_var_flt(iptr->dst, d);
1819                         break;
1820
1821                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1822
1823                         var_to_reg_int(s1, src, REG_ITMP1);
1824                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1825                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1826                         store_reg_to_var_flt(iptr->dst, d);
1827                         break;
1828
1829                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1830
1831                         var_to_reg_int(s1, src, REG_ITMP1);
1832                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1833                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1834                         store_reg_to_var_flt(iptr->dst, d);
1835                         break;
1836
1837                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1838
1839                         var_to_reg_int(s1, src, REG_ITMP1);
1840                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1841                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1842                         store_reg_to_var_flt(iptr->dst, d);
1843                         break;
1844
1845                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1846
1847                         var_to_reg_int(s1, src, REG_ITMP1);
1848                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1849                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1850                         store_reg_to_var_flt(iptr->dst, d);
1851                         break;
1852
1853                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1854
1855                         var_to_reg_flt(s1, src, REG_FTMP1);
1856                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1857                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1858                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1859                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1860                         x86_64_jcc(cd, X86_64_CC_NE, a);
1861                         M_FLTMOVE(s1, REG_FTMP1);
1862                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1863                         x86_64_call_reg(cd, REG_ITMP2);
1864                         M_INTMOVE(REG_RESULT, d);
1865                         store_reg_to_var_int(iptr->dst, d);
1866                         break;
1867
1868                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1869
1870                         var_to_reg_flt(s1, src, REG_FTMP1);
1871                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1872                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1873                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1874                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1875                         x86_64_jcc(cd, X86_64_CC_NE, a);
1876                         M_FLTMOVE(s1, REG_FTMP1);
1877                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1878                         x86_64_call_reg(cd, REG_ITMP2);
1879                         M_INTMOVE(REG_RESULT, d);
1880                         store_reg_to_var_int(iptr->dst, d);
1881                         break;
1882
1883                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1884
1885                         var_to_reg_flt(s1, src, REG_FTMP1);
1886                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1887                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1888                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1889                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1890                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1891                         x86_64_jcc(cd, X86_64_CC_NE, a);
1892                         M_FLTMOVE(s1, REG_FTMP1);
1893                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1894                         x86_64_call_reg(cd, REG_ITMP2);
1895                         M_INTMOVE(REG_RESULT, d);
1896                         store_reg_to_var_int(iptr->dst, d);
1897                         break;
1898
1899                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1900
1901                         var_to_reg_flt(s1, src, REG_FTMP1);
1902                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1903                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1904                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1905                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1906                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1907                         x86_64_jcc(cd, X86_64_CC_NE, a);
1908                         M_FLTMOVE(s1, REG_FTMP1);
1909                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1910                         x86_64_call_reg(cd, REG_ITMP2);
1911                         M_INTMOVE(REG_RESULT, d);
1912                         store_reg_to_var_int(iptr->dst, d);
1913                         break;
1914
1915                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1916
1917                         var_to_reg_flt(s1, src, REG_FTMP1);
1918                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1919                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1920                         store_reg_to_var_flt(iptr->dst, d);
1921                         break;
1922
1923                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1924
1925                         var_to_reg_flt(s1, src, REG_FTMP1);
1926                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1927                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1928                         store_reg_to_var_flt(iptr->dst, d);
1929                         break;
1930
1931                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1932                                           /* == => 0, < => 1, > => -1 */
1933
1934                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1935                         var_to_reg_flt(s2, src, REG_FTMP2);
1936                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1937                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1938                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1939                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1940                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1941                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1942                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1943                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1944                         store_reg_to_var_int(iptr->dst, d);
1945                         break;
1946
1947                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1948                                           /* == => 0, < => 1, > => -1 */
1949
1950                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1951                         var_to_reg_flt(s2, src, REG_FTMP2);
1952                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1953                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1954                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1955                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1956                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1957                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1958                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1959                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1960                         store_reg_to_var_int(iptr->dst, d);
1961                         break;
1962
1963                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1964                                           /* == => 0, < => 1, > => -1 */
1965
1966                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1967                         var_to_reg_flt(s2, src, REG_FTMP2);
1968                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1969                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1970                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1971                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1972                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1973                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1974                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1975                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1976                         store_reg_to_var_int(iptr->dst, d);
1977                         break;
1978
1979                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1980                                           /* == => 0, < => 1, > => -1 */
1981
1982                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1983                         var_to_reg_flt(s2, src, REG_FTMP2);
1984                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1985                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1986                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1987                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1988                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1989                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1990                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1991                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1992                         store_reg_to_var_int(iptr->dst, d);
1993                         break;
1994
1995
1996                 /* memory operations **************************************************/
1997
1998                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1999
2000                         var_to_reg_int(s1, src, REG_ITMP1);
2001                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2002                         gen_nullptr_check(s1);
2003                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
2004                         store_reg_to_var_int(iptr->dst, d);
2005                         break;
2006
2007                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2008
2009                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2010                         var_to_reg_int(s2, src, REG_ITMP2);
2011                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2012                         if (iptr->op1 == 0) {
2013                                 gen_nullptr_check(s1);
2014                                 gen_bound_check;
2015                         }
2016                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2017                         store_reg_to_var_int(iptr->dst, d);
2018                         break;
2019
2020                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2021
2022                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2023                         var_to_reg_int(s2, src, REG_ITMP2);
2024                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2025                         if (iptr->op1 == 0) {
2026                                 gen_nullptr_check(s1);
2027                                 gen_bound_check;
2028                         }
2029                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2030                         store_reg_to_var_int(iptr->dst, d);
2031                         break;
2032
2033                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2034
2035                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2036                         var_to_reg_int(s2, src, REG_ITMP2);
2037                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2038                         if (iptr->op1 == 0) {
2039                                 gen_nullptr_check(s1);
2040                                 gen_bound_check;
2041                         }
2042                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2043                         store_reg_to_var_int(iptr->dst, d);
2044                         break;
2045
2046                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2047
2048                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2049                         var_to_reg_int(s2, src, REG_ITMP2);
2050                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2051                         if (iptr->op1 == 0) {
2052                                 gen_nullptr_check(s1);
2053                                 gen_bound_check;
2054                         }
2055                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2056                         store_reg_to_var_flt(iptr->dst, d);
2057                         break;
2058
2059                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2060
2061                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2062                         var_to_reg_int(s2, src, REG_ITMP2);
2063                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2064                         if (iptr->op1 == 0) {
2065                                 gen_nullptr_check(s1);
2066                                 gen_bound_check;
2067                         }
2068                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2069                         store_reg_to_var_flt(iptr->dst, d);
2070                         break;
2071
2072                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2073
2074                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2075                         var_to_reg_int(s2, src, REG_ITMP2);
2076                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2077                         if (iptr->op1 == 0) {
2078                                 gen_nullptr_check(s1);
2079                                 gen_bound_check;
2080                         }
2081                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2082                         store_reg_to_var_int(iptr->dst, d);
2083                         break;
2084
2085                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2086
2087                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2088                         var_to_reg_int(s2, src, REG_ITMP2);
2089                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2090                         if (iptr->op1 == 0) {
2091                                 gen_nullptr_check(s1);
2092                                 gen_bound_check;
2093                         }
2094                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2095                         store_reg_to_var_int(iptr->dst, d);
2096                         break;
2097
2098                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2099
2100                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2101                         var_to_reg_int(s2, src, REG_ITMP2);
2102                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2103                         if (iptr->op1 == 0) {
2104                                 gen_nullptr_check(s1);
2105                                 gen_bound_check;
2106                         }
2107                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2108                         store_reg_to_var_int(iptr->dst, d);
2109                         break;
2110
2111
2112                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2113
2114                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2115                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2116                         if (iptr->op1 == 0) {
2117                                 gen_nullptr_check(s1);
2118                                 gen_bound_check;
2119                         }
2120                         var_to_reg_int(s3, src, REG_ITMP3);
2121                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2122                         break;
2123
2124                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2125
2126                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2127                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2128                         if (iptr->op1 == 0) {
2129                                 gen_nullptr_check(s1);
2130                                 gen_bound_check;
2131                         }
2132                         var_to_reg_int(s3, src, REG_ITMP3);
2133                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2134                         break;
2135
2136                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2137
2138                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2139                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2140                         if (iptr->op1 == 0) {
2141                                 gen_nullptr_check(s1);
2142                                 gen_bound_check;
2143                         }
2144                         var_to_reg_int(s3, src, REG_ITMP3);
2145                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2146                         break;
2147
2148                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2149
2150                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2151                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2152                         if (iptr->op1 == 0) {
2153                                 gen_nullptr_check(s1);
2154                                 gen_bound_check;
2155                         }
2156                         var_to_reg_flt(s3, src, REG_FTMP3);
2157                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2158                         break;
2159
2160                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2161
2162                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2163                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2164                         if (iptr->op1 == 0) {
2165                                 gen_nullptr_check(s1);
2166                                 gen_bound_check;
2167                         }
2168                         var_to_reg_flt(s3, src, REG_FTMP3);
2169                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2170                         break;
2171
2172                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2173
2174                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2175                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2176                         if (iptr->op1 == 0) {
2177                                 gen_nullptr_check(s1);
2178                                 gen_bound_check;
2179                         }
2180                         var_to_reg_int(s3, src, REG_ITMP3);
2181                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2182                         break;
2183
2184                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2185
2186                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2187                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2188                         if (iptr->op1 == 0) {
2189                                 gen_nullptr_check(s1);
2190                                 gen_bound_check;
2191                         }
2192                         var_to_reg_int(s3, src, REG_ITMP3);
2193                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2194                         break;
2195
2196                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2197
2198                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2199                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2200                         if (iptr->op1 == 0) {
2201                                 gen_nullptr_check(s1);
2202                                 gen_bound_check;
2203                         }
2204                         var_to_reg_int(s3, src, REG_ITMP3);
2205                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2206                         break;
2207
2208                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2209
2210                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2211                         var_to_reg_int(s2, src, REG_ITMP2);
2212                         if (iptr->op1 == 0) {
2213                                 gen_nullptr_check(s1);
2214                                 gen_bound_check;
2215                         }
2216                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2217                         break;
2218
2219                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2220
2221                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2222                         var_to_reg_int(s2, src, REG_ITMP2);
2223                         if (iptr->op1 == 0) {
2224                                 gen_nullptr_check(s1);
2225                                 gen_bound_check;
2226                         }
2227
2228                         if (x86_64_is_imm32(iptr->val.l)) {
2229                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2230
2231                         } else {
2232                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2233                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2234                         }
2235                         break;
2236
2237                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2238
2239                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2240                         var_to_reg_int(s2, src, REG_ITMP2);
2241                         if (iptr->op1 == 0) {
2242                                 gen_nullptr_check(s1);
2243                                 gen_bound_check;
2244                         }
2245                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2246                         break;
2247
2248                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2249
2250                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2251                         var_to_reg_int(s2, src, REG_ITMP2);
2252                         if (iptr->op1 == 0) {
2253                                 gen_nullptr_check(s1);
2254                                 gen_bound_check;
2255                         }
2256                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2257                         break;
2258
2259                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2260
2261                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2262                         var_to_reg_int(s2, src, REG_ITMP2);
2263                         if (iptr->op1 == 0) {
2264                                 gen_nullptr_check(s1);
2265                                 gen_bound_check;
2266                         }
2267                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2268                         break;
2269
2270                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2271
2272                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2273                         var_to_reg_int(s2, src, REG_ITMP2);
2274                         if (iptr->op1 == 0) {
2275                                 gen_nullptr_check(s1);
2276                                 gen_bound_check;
2277                         }
2278                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2279                         break;
2280
2281
2282                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2283                                       /* op1 = type, val.a = field address            */
2284
2285                         /* if class isn't yet initialized, do it */
2286                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2287                                 /* call helper function which patches this code */
2288                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2289                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2290                                 x86_64_call_reg(cd, REG_ITMP2);
2291                         }
2292
2293                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2294                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2295                         switch (iptr->op1) {
2296                         case TYPE_INT:
2297                                 var_to_reg_int(s2, src, REG_ITMP1);
2298                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2299                                 break;
2300                         case TYPE_LNG:
2301                         case TYPE_ADR:
2302                                 var_to_reg_int(s2, src, REG_ITMP1);
2303                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2304                                 break;
2305                         case TYPE_FLT:
2306                                 var_to_reg_flt(s2, src, REG_FTMP1);
2307                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2308                                 break;
2309                         case TYPE_DBL:
2310                                 var_to_reg_flt(s2, src, REG_FTMP1);
2311                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2312                                 break;
2313                         default: panic("internal error");
2314                         }
2315                         break;
2316
2317                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2318                                       /* op1 = type, val.a = field address            */
2319
2320                         /* if class isn't yet initialized, do it */
2321                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2322                                 /* call helper function which patches this code */
2323                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2324                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2325                                 x86_64_call_reg(cd, REG_ITMP2);
2326                         }
2327
2328                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2329                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2330                         switch (iptr->op1) {
2331                         case TYPE_INT:
2332                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2333                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2334                                 store_reg_to_var_int(iptr->dst, d);
2335                                 break;
2336                         case TYPE_LNG:
2337                         case TYPE_ADR:
2338                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2339                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2340                                 store_reg_to_var_int(iptr->dst, d);
2341                                 break;
2342                         case TYPE_FLT:
2343                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2344                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2345                                 store_reg_to_var_flt(iptr->dst, d);
2346                                 break;
2347                         case TYPE_DBL:
2348                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2349                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2350                                 store_reg_to_var_flt(iptr->dst, d);
2351                                 break;
2352                         default: panic("internal error");
2353                         }
2354                         break;
2355
2356                 case ICMD_PUTFIELD:   /* ..., value  ==> ...                          */
2357                                       /* op1 = type, val.i = field offset             */
2358
2359                         /* if class isn't yet initialized, do it */
2360                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2361                                 /* call helper function which patches this code */
2362                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2363                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2364                                 x86_64_call_reg(cd, REG_ITMP2);
2365                         }
2366
2367                         a = ((fieldinfo *)(iptr->val.a))->offset;
2368                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2369                         switch (iptr->op1) {
2370                                 case TYPE_INT:
2371                                         var_to_reg_int(s2, src, REG_ITMP2);
2372                                         gen_nullptr_check(s1);
2373                                         x86_64_movl_reg_membase(cd, s2, s1, a);
2374                                         break;
2375                                 case TYPE_LNG:
2376                                 case TYPE_ADR:
2377                                         var_to_reg_int(s2, src, REG_ITMP2);
2378                                         gen_nullptr_check(s1);
2379                                         x86_64_mov_reg_membase(cd, s2, s1, a);
2380                                         break;
2381                                 case TYPE_FLT:
2382                                         var_to_reg_flt(s2, src, REG_FTMP2);
2383                                         gen_nullptr_check(s1);
2384                                         x86_64_movss_reg_membase(cd, s2, s1, a);
2385                                         break;
2386                                 case TYPE_DBL:
2387                                         var_to_reg_flt(s2, src, REG_FTMP2);
2388                                         gen_nullptr_check(s1);
2389                                         x86_64_movsd_reg_membase(cd, s2, s1, a);
2390                                         break;
2391                                 default: panic ("internal error");
2392                                 }
2393                         break;
2394
2395                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2396                                       /* op1 = type, val.i = field offset             */
2397
2398                         a = ((fieldinfo *)(iptr->val.a))->offset;
2399                         var_to_reg_int(s1, src, REG_ITMP1);
2400                         switch (iptr->op1) {
2401                                 case TYPE_INT:
2402                                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2403                                         gen_nullptr_check(s1);
2404                                         x86_64_movl_membase_reg(cd, s1, a, d);
2405                                         store_reg_to_var_int(iptr->dst, d);
2406                                         break;
2407                                 case TYPE_LNG:
2408                                 case TYPE_ADR:
2409                                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2410                                         gen_nullptr_check(s1);
2411                                         x86_64_mov_membase_reg(cd, s1, a, d);
2412                                         store_reg_to_var_int(iptr->dst, d);
2413                                         break;
2414                                 case TYPE_FLT:
2415                                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2416                                         gen_nullptr_check(s1);
2417                                         x86_64_movss_membase_reg(cd, s1, a, d);
2418                                         store_reg_to_var_flt(iptr->dst, d);
2419                                         break;
2420                                 case TYPE_DBL:
2421                                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2422                                         gen_nullptr_check(s1);
2423                                         x86_64_movsd_membase_reg(cd, s1, a, d);
2424                                         store_reg_to_var_flt(iptr->dst, d);
2425                                         break;
2426                                 default: panic ("internal error");
2427                                 }
2428                         break;
2429
2430
2431                 /* branch operations **************************************************/
2432
2433                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2434
2435                         var_to_reg_int(s1, src, REG_ITMP1);
2436                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2437
2438                         x86_64_call_imm(cd, 0); /* passing exception pointer                  */
2439                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2440
2441                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
2442                         x86_64_jmp_reg(cd, REG_ITMP3);
2443                         ALIGNCODENOP;
2444                         break;
2445
2446                 case ICMD_GOTO:         /* ... ==> ...                                */
2447                                         /* op1 = target JavaVM pc                     */
2448
2449                         x86_64_jmp_imm(cd, 0);
2450                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2451                         ALIGNCODENOP;
2452                         break;
2453
2454                 case ICMD_JSR:          /* ... ==> ...                                */
2455                                         /* op1 = target JavaVM pc                     */
2456
2457                         x86_64_call_imm(cd, 0);
2458                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2459                         break;
2460
2461                 case ICMD_RET:          /* ... ==> ...                                */
2462                                         /* op1 = local variable                       */
2463
2464                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2465                         var_to_reg_int(s1, var, REG_ITMP1);
2466                         x86_64_jmp_reg(cd, s1);
2467                         break;
2468
2469                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2470                                         /* op1 = target JavaVM pc                     */
2471
2472                         if (src->flags & INMEMORY) {
2473                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2474
2475                         } else {
2476                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2477                         }
2478                         x86_64_jcc(cd, X86_64_CC_E, 0);
2479                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2480                         break;
2481
2482                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2483                                         /* op1 = target JavaVM pc                     */
2484
2485                         if (src->flags & INMEMORY) {
2486                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2487
2488                         } else {
2489                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2490                         }
2491                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2492                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2493                         break;
2494
2495                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2496                                         /* op1 = target JavaVM pc, val.i = constant   */
2497
2498                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2499                         break;
2500
2501                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2502                                         /* op1 = target JavaVM pc, val.i = constant   */
2503
2504                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2505                         break;
2506
2507                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2508                                         /* op1 = target JavaVM pc, val.i = constant   */
2509
2510                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2511                         break;
2512
2513                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2514                                         /* op1 = target JavaVM pc, val.i = constant   */
2515
2516                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2517                         break;
2518
2519                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2520                                         /* op1 = target JavaVM pc, val.i = constant   */
2521
2522                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2523                         break;
2524
2525                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2526                                         /* op1 = target JavaVM pc, val.i = constant   */
2527
2528                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2529                         break;
2530
2531                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2532                                         /* op1 = target JavaVM pc, val.l = constant   */
2533
2534                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2535                         break;
2536
2537                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2538                                         /* op1 = target JavaVM pc, val.l = constant   */
2539
2540                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2541                         break;
2542
2543                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2544                                         /* op1 = target JavaVM pc, val.l = constant   */
2545
2546                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2547                         break;
2548
2549                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2550                                         /* op1 = target JavaVM pc, val.l = constant   */
2551
2552                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2553                         break;
2554
2555                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2556                                         /* op1 = target JavaVM pc, val.l = constant   */
2557
2558                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2559                         break;
2560
2561                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2562                                         /* op1 = target JavaVM pc, val.l = constant   */
2563
2564                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2565                         break;
2566
2567                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2568                                         /* op1 = target JavaVM pc                     */
2569
2570                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2571                         break;
2572
2573                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2574                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2575
2576                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2577                         break;
2578
2579                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2580                                         /* op1 = target JavaVM pc                     */
2581
2582                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2583                         break;
2584
2585                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2586                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2587
2588                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2589                         break;
2590
2591                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2592                                         /* op1 = target JavaVM pc                     */
2593
2594                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2595                         break;
2596
2597                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2598                                     /* op1 = target JavaVM pc                     */
2599
2600                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2601                         break;
2602
2603                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2604                                         /* op1 = target JavaVM pc                     */
2605
2606                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2607                         break;
2608
2609                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2610                                 /* op1 = target JavaVM pc                     */
2611
2612                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2613                         break;
2614
2615                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2616                                         /* op1 = target JavaVM pc                     */
2617
2618                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2619                         break;
2620
2621                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2622                                         /* op1 = target JavaVM pc                     */
2623
2624                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2625                         break;
2626
2627                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2628                                         /* op1 = target JavaVM pc                     */
2629
2630                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2631                         break;
2632
2633                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2634                                     /* op1 = target JavaVM pc                     */
2635
2636                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2637                         break;
2638
2639                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2640
2641                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2642                         break;
2643
2644                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2645                                         /* val.i = constant                           */
2646
2647                         var_to_reg_int(s1, src, REG_ITMP1);
2648                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2649                         s3 = iptr->val.i;
2650                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2651                                 if (s1 == d) {
2652                                         M_INTMOVE(s1, REG_ITMP1);
2653                                         s1 = REG_ITMP1;
2654                                 }
2655                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2656                         }
2657                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2658                         x86_64_testl_reg_reg(cd, s1, s1);
2659                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2660                         store_reg_to_var_int(iptr->dst, d);
2661                         break;
2662
2663                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2664                                         /* val.i = constant                           */
2665
2666                         var_to_reg_int(s1, src, REG_ITMP1);
2667                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2668                         s3 = iptr->val.i;
2669                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2670                                 if (s1 == d) {
2671                                         M_INTMOVE(s1, REG_ITMP1);
2672                                         s1 = REG_ITMP1;
2673                                 }
2674                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2675                         }
2676                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2677                         x86_64_testl_reg_reg(cd, s1, s1);
2678                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2679                         store_reg_to_var_int(iptr->dst, d);
2680                         break;
2681
2682                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2683                                         /* val.i = constant                           */
2684
2685                         var_to_reg_int(s1, src, REG_ITMP1);
2686                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2687                         s3 = iptr->val.i;
2688                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2689                                 if (s1 == d) {
2690                                         M_INTMOVE(s1, REG_ITMP1);
2691                                         s1 = REG_ITMP1;
2692                                 }
2693                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2694                         }
2695                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2696                         x86_64_testl_reg_reg(cd, s1, s1);
2697                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2698                         store_reg_to_var_int(iptr->dst, d);
2699                         break;
2700
2701                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2702                                         /* val.i = constant                           */
2703
2704                         var_to_reg_int(s1, src, REG_ITMP1);
2705                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2706                         s3 = iptr->val.i;
2707                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2708                                 if (s1 == d) {
2709                                         M_INTMOVE(s1, REG_ITMP1);
2710                                         s1 = REG_ITMP1;
2711                                 }
2712                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2713                         }
2714                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2715                         x86_64_testl_reg_reg(cd, s1, s1);
2716                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2717                         store_reg_to_var_int(iptr->dst, d);
2718                         break;
2719
2720                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2721                                         /* val.i = constant                           */
2722
2723                         var_to_reg_int(s1, src, REG_ITMP1);
2724                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2725                         s3 = iptr->val.i;
2726                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2727                                 if (s1 == d) {
2728                                         M_INTMOVE(s1, REG_ITMP1);
2729                                         s1 = REG_ITMP1;
2730                                 }
2731                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2732                         }
2733                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2734                         x86_64_testl_reg_reg(cd, s1, s1);
2735                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2736                         store_reg_to_var_int(iptr->dst, d);
2737                         break;
2738
2739                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2740                                         /* val.i = constant                           */
2741
2742                         var_to_reg_int(s1, src, REG_ITMP1);
2743                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2744                         s3 = iptr->val.i;
2745                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2746                                 if (s1 == d) {
2747                                         M_INTMOVE(s1, REG_ITMP1);
2748                                         s1 = REG_ITMP1;
2749                                 }
2750                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2751                         }
2752                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2753                         x86_64_testl_reg_reg(cd, s1, s1);
2754                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2755                         store_reg_to_var_int(iptr->dst, d);
2756                         break;
2757
2758
2759                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2760                 case ICMD_LRETURN:
2761                 case ICMD_ARETURN:
2762
2763                         var_to_reg_int(s1, src, REG_RESULT);
2764                         M_INTMOVE(s1, REG_RESULT);
2765
2766 #if defined(USE_THREADS)
2767                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2768                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2769                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->maxmemuse * 8);
2770                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2771                                 x86_64_call_reg(cd, REG_ITMP1);
2772                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_RESULT);
2773                         }
2774 #endif
2775
2776                         goto nowperformreturn;
2777
2778                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2779                 case ICMD_DRETURN:
2780
2781                         var_to_reg_flt(s1, src, REG_FRESULT);
2782                         M_FLTMOVE(s1, REG_FRESULT);
2783
2784 #if defined(USE_THREADS)
2785                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2786                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2787                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->maxmemuse * 8);
2788                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2789                                 x86_64_call_reg(cd, REG_ITMP1);
2790                                 x86_64_movq_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_FRESULT);
2791                         }
2792 #endif
2793
2794                         goto nowperformreturn;
2795
2796                 case ICMD_RETURN:      /* ...  ==> ...                                */
2797
2798 #if defined(USE_THREADS)
2799                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2800                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2801                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2802                                 x86_64_call_reg(cd, REG_ITMP1);
2803                         }
2804 #endif
2805
2806 nowperformreturn:
2807                         {
2808                         s4 i, p;
2809
2810                         p = parentargs_base;
2811
2812                         /* call trace function */
2813                         if (runverbose) {
2814                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2815
2816                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2817                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2818
2819                                 x86_64_mov_imm_reg(cd, (s8) m, rd->argintregs[0]);
2820                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2821                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2822                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2823
2824                                 x86_64_mov_imm_reg(cd, (s8) builtin_displaymethodstop, REG_ITMP1);
2825                                 x86_64_call_reg(cd, REG_ITMP1);
2826
2827                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2828                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2829
2830                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2831                         }
2832
2833                         /* restore saved registers                                        */
2834                         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
2835                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
2836                         }
2837                         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
2838                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
2839                         }
2840
2841                         /* deallocate stack                                               */
2842                         if (parentargs_base) {
2843                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2844                         }
2845
2846                         x86_64_ret(cd);
2847                         ALIGNCODENOP;
2848                         }
2849                         break;
2850
2851
2852                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2853                         {
2854                                 s4 i, l, *s4ptr;
2855                                 void **tptr;
2856
2857                                 tptr = (void **) iptr->target;
2858
2859                                 s4ptr = iptr->val.a;
2860                                 l = s4ptr[1];                          /* low     */
2861                                 i = s4ptr[2];                          /* high    */
2862
2863                                 var_to_reg_int(s1, src, REG_ITMP1);
2864                                 M_INTMOVE(s1, REG_ITMP1);
2865                                 if (l != 0) {
2866                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2867                                 }
2868                                 i = i - l + 1;
2869
2870                 /* range check */
2871                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2872                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2873
2874                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2875                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2876
2877                                 /* build jump table top down and use address of lowest entry */
2878
2879                 /* s4ptr += 3 + i; */
2880                                 tptr += i;
2881
2882                                 while (--i >= 0) {
2883                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
2884                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2885                                         --tptr;
2886                                 }
2887
2888                                 /* length of dataseg after last dseg_addtarget is used by load */
2889
2890                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2891                                 dseg_adddata(cd, cd->mcodeptr);
2892                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2893                                 x86_64_jmp_reg(cd, REG_ITMP1);
2894                                 ALIGNCODENOP;
2895                         }
2896                         break;
2897
2898
2899                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2900                         {
2901                                 s4 i, l, val, *s4ptr;
2902                                 void **tptr;
2903
2904                                 tptr = (void **) iptr->target;
2905
2906                                 s4ptr = iptr->val.a;
2907                                 l = s4ptr[0];                          /* default  */
2908                                 i = s4ptr[1];                          /* count    */
2909
2910                                 MCODECHECK((i<<2)+8);
2911                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
2912                                 while (--i >= 0) {
2913                                         s4ptr += 2;
2914                                         ++tptr;
2915
2916                                         val = s4ptr[0];
2917                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
2918                                         x86_64_jcc(cd, X86_64_CC_E, 0);
2919                                         /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
2920                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2921                                 }
2922
2923                                 x86_64_jmp_imm(cd, 0);
2924                                 /* codegen_addreference(cd, BlockPtrOfPC(l), cd->mcodeptr); */
2925
2926                                 tptr = (void **) iptr->target;
2927                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2928
2929                                 ALIGNCODENOP;
2930                         }
2931                         break;
2932
2933
2934                 case ICMD_BUILTIN3:     /* ..., arg1, arg2, arg3 ==> ...              */
2935                                         /* op1 = return type, val.a = function pointer*/
2936                         s3 = 3;
2937                         goto gen_method;
2938
2939                 case ICMD_BUILTIN2:     /* ..., arg1, arg2 ==> ...                    */
2940                                         /* op1 = return type, val.a = function pointer*/
2941                         s3 = 2;
2942                         goto gen_method;
2943
2944                 case ICMD_BUILTIN1:     /* ..., arg1 ==> ...                          */
2945                                         /* op1 = return type, val.a = function pointer*/
2946                         s3 = 1;
2947                         goto gen_method;
2948
2949                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2950                                         /* op1 = arg count, val.a = method pointer    */
2951
2952                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2953                                         /* op1 = arg count, val.a = method pointer    */
2954
2955                 case ICMD_INVOKEVIRTUAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2956                                         /* op1 = arg count, val.a = method pointer    */
2957
2958                 case ICMD_INVOKEINTERFACE:/*.., objectref, [arg1, [arg2 ...]] ==> ... */
2959                                         /* op1 = arg count, val.a = method pointer    */
2960
2961                         s3 = iptr->op1;
2962
2963 gen_method: {
2964                         methodinfo   *lm;
2965                         classinfo    *ci;
2966                         stackptr      tmpsrc;
2967                         s4 iarg, farg;
2968
2969                         MCODECHECK((s3 << 1) + 64);
2970
2971                         tmpsrc = src;
2972                         s2 = s3;
2973                         iarg = 0;
2974                         farg = 0;
2975
2976                         /* copy arguments to registers or stack location                  */
2977                         for (; --s3 >= 0; src = src->prev) {
2978                                 IS_INT_LNG_TYPE(src->type) ? iarg++ : farg++;
2979                         }
2980
2981                         src = tmpsrc;
2982                         s3 = s2;
2983
2984                         s2 = (iarg > INT_ARG_CNT) ? iarg - INT_ARG_CNT : 0 + (farg > FLT_ARG_CNT) ? farg - FLT_ARG_CNT : 0;
2985
2986                         for (; --s3 >= 0; src = src->prev) {
2987                                 IS_INT_LNG_TYPE(src->type) ? iarg-- : farg--;
2988                                 if (src->varkind == ARGVAR) {
2989                                         if (IS_INT_LNG_TYPE(src->type)) {
2990                                                 if (iarg >= INT_ARG_CNT) {
2991                                                         s2--;
2992                                                 }
2993                                         } else {
2994                                                 if (farg >= FLT_ARG_CNT) {
2995                                                         s2--;
2996                                                 }
2997                                         }
2998                                         continue;
2999                                 }
3000
3001                                 if (IS_INT_LNG_TYPE(src->type)) {
3002                                         if (iarg < INT_ARG_CNT) {
3003                                                 s1 = rd->argintregs[iarg];
3004                                                 var_to_reg_int(d, src, s1);
3005                                                 M_INTMOVE(d, s1);
3006
3007                                         } else {
3008                                                 var_to_reg_int(d, src, REG_ITMP1);
3009                                                 s2--;
3010                                                 x86_64_mov_reg_membase(cd, d, REG_SP, s2 * 8);
3011                                         }
3012
3013                                 } else {
3014                                         if (farg < FLT_ARG_CNT) {
3015                                                 s1 = rd->argfltregs[farg];
3016                                                 var_to_reg_flt(d, src, s1);
3017                                                 M_FLTMOVE(d, s1);
3018
3019                                         } else {
3020                                                 var_to_reg_flt(d, src, REG_FTMP1);
3021                                                 s2--;
3022                                                 x86_64_movq_reg_membase(cd, d, REG_SP, s2 * 8);
3023                                         }
3024                                 }
3025                         } /* end of for */
3026
3027                         lm = iptr->val.a;
3028                         switch (iptr->opc) {
3029                                 case ICMD_BUILTIN3:
3030                                 case ICMD_BUILTIN2:
3031                                 case ICMD_BUILTIN1:
3032
3033                                         a = (s8) lm;
3034                                         d = iptr->op1;
3035
3036                                         x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3037                                         x86_64_call_reg(cd, REG_ITMP1);
3038                                         break;
3039
3040                                 case ICMD_INVOKESTATIC:
3041
3042                                         a = (s8) lm->stubroutine;
3043                                         d = lm->returntype;
3044
3045                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3046                                         x86_64_call_reg(cd, REG_ITMP2);
3047                                         break;
3048
3049                                 case ICMD_INVOKESPECIAL:
3050
3051                                         a = (s8) lm->stubroutine;
3052                                         d = lm->returntype;
3053
3054                                         gen_nullptr_check(rd->argintregs[0]);    /* first argument contains pointer */
3055                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); /* access memory for hardware nullptr */
3056                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3057                                         x86_64_call_reg(cd, REG_ITMP2);
3058                                         break;
3059
3060                                 case ICMD_INVOKEVIRTUAL:
3061
3062                                         d = lm->returntype;
3063
3064                                         gen_nullptr_check(rd->argintregs[0]);
3065                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3066                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, OFFSET(vftbl_t, table[0]) + sizeof(methodptr) * lm->vftblindex, REG_ITMP1);
3067                                         x86_64_call_reg(cd, REG_ITMP1);
3068                                         break;
3069
3070                                 case ICMD_INVOKEINTERFACE:
3071
3072                                         ci = lm->class;
3073                                         d = lm->returntype;
3074
3075                                         gen_nullptr_check(rd->argintregs[0]);
3076                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3077                                         x86_64_mov_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, interfacetable[0]) - sizeof(methodptr) * ci->index, REG_ITMP2);
3078                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, sizeof(methodptr) * (lm - ci->methods), REG_ITMP1);
3079                                         x86_64_call_reg(cd, REG_ITMP1);
3080                                         break;
3081
3082                                 default:
3083                                         d = 0;
3084                                         error("Unkown ICMD-Command: %d", iptr->opc);
3085                                 }
3086
3087                         /* d contains return type */
3088
3089                         if (d != TYPE_VOID) {
3090                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3091                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3092                                         M_INTMOVE(REG_RESULT, s1);
3093                                         store_reg_to_var_int(iptr->dst, s1);
3094
3095                                 } else {
3096                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3097                                         M_FLTMOVE(REG_FRESULT, s1);
3098                                         store_reg_to_var_flt(iptr->dst, s1);
3099                                 }
3100                         }
3101                         }
3102                         break;
3103
3104
3105                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3106
3107                                       /* op1:   0 == array, 1 == class                */
3108                                       /* val.a: (classinfo*) superclass               */
3109
3110 /*          superclass is an interface:
3111  *
3112  *          return (sub != NULL) &&
3113  *                 (sub->vftbl->interfacetablelength > super->index) &&
3114  *                 (sub->vftbl->interfacetable[-super->index] != NULL);
3115  *
3116  *          superclass is a class:
3117  *
3118  *          return ((sub != NULL) && (0
3119  *                  <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3120  *                  super->vftbl->diffvall));
3121  */
3122
3123                         {
3124                         classinfo *super = (classinfo*) iptr->val.a;
3125
3126 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3127             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3128 #endif
3129
3130                         var_to_reg_int(s1, src, REG_ITMP1);
3131                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3132                         if (s1 == d) {
3133                                 M_INTMOVE(s1, REG_ITMP1);
3134                                 s1 = REG_ITMP1;
3135                         }
3136                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3137                         if (iptr->op1) {                               /* class/interface */
3138                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3139                                         x86_64_test_reg_reg(cd, s1, s1);
3140
3141                                         /* TODO: clean up this calculation */
3142                                         a = 3;    /* mov_membase_reg */
3143                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3144
3145                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3146                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3147
3148                                         a += 3;    /* sub */
3149                                         CALCIMMEDIATEBYTES(a, super->index);
3150
3151                                         a += 3;    /* test */
3152
3153                                         a += 6;    /* jcc */
3154                                         a += 3;    /* mov_membase_reg */
3155                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3156
3157                                         a += 3;    /* test */
3158                                         a += 4;    /* setcc */
3159
3160                                         x86_64_jcc(cd, X86_64_CC_E, a);
3161
3162                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3163                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3164                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3165                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3166
3167                                         /* TODO: clean up this calculation */
3168                                         a = 0;
3169                                         a += 3;    /* mov_membase_reg */
3170                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3171
3172                                         a += 3;    /* test */
3173                                         a += 4;    /* setcc */
3174
3175                                         x86_64_jcc(cd, X86_64_CC_LE, a);
3176                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP1);
3177                                         x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3178                                         x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3179
3180                                 } else {                                   /* class           */
3181                                         x86_64_test_reg_reg(cd, s1, s1);
3182
3183                                         /* TODO: clean up this calculation */
3184                                         a = 3;    /* mov_membase_reg */
3185                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3186
3187                                         a += 10;   /* mov_imm_reg */
3188
3189                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3190                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3191
3192                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3193                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3194
3195                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3196                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3197
3198                                         a += 3;    /* sub */
3199                                         a += 3;    /* xor */
3200                                         a += 3;    /* cmp */
3201                                         a += 4;    /* setcc */
3202
3203                                         x86_64_jcc(cd, X86_64_CC_E, a);
3204
3205                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3206                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3207 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3208                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3209 #endif
3210                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3211                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3212                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3213 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3214                     codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3215 #endif
3216                                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3217                                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3218                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3219                                         x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3220                                 }
3221                         }
3222                         else
3223                                 panic("internal error: no inlined array instanceof");
3224                         }
3225                         store_reg_to_var_int(iptr->dst, d);
3226                         break;
3227
3228                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3229
3230                                       /* op1:   0 == array, 1 == class                */
3231                                       /* val.a: (classinfo*) superclass               */
3232
3233 /*          superclass is an interface:
3234  *
3235  *          OK if ((sub == NULL) ||
3236  *                 (sub->vftbl->interfacetablelength > super->index) &&
3237  *                 (sub->vftbl->interfacetable[-super->index] != NULL));
3238  *
3239  *          superclass is a class:
3240  *
3241  *          OK if ((sub == NULL) || (0
3242  *                 <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3243  *                 super->vftbl->diffvall));
3244  */
3245
3246                         {
3247                         classinfo *super = (classinfo*) iptr->val.a;
3248
3249 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3250             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3251 #endif
3252                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3253                         var_to_reg_int(s1, src, d);
3254                         if (iptr->op1) {                               /* class/interface */
3255                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3256                                         x86_64_test_reg_reg(cd, s1, s1);
3257
3258                                         /* TODO: clean up this calculation */
3259                                         a = 3;    /* mov_membase_reg */
3260                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3261
3262                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3263                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3264
3265                                         a += 3;    /* sub */
3266                                         CALCIMMEDIATEBYTES(a, super->index);
3267
3268                                         a += 3;    /* test */
3269                                         a += 6;    /* jcc */
3270
3271                                         a += 3;    /* mov_membase_reg */
3272                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3273
3274                                         a += 3;    /* test */
3275                                         a += 6;    /* jcc */
3276
3277                                         x86_64_jcc(cd, X86_64_CC_E, a);
3278
3279                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3280                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3281                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3282                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3283                                         x86_64_jcc(cd, X86_64_CC_LE, 0);
3284                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3285                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP2);
3286                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3287                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3288                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3289
3290                                 } else {                                     /* class           */
3291                                         x86_64_test_reg_reg(cd, s1, s1);
3292
3293                                         /* TODO: clean up this calculation */
3294                                         a = 3;    /* mov_membase_reg */
3295                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3296                                         a += 10;   /* mov_imm_reg */
3297                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3298                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3299
3300                                         if (d != REG_ITMP3) {
3301                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3302                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3303                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3304                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3305                                                 a += 3;    /* sub */
3306
3307                                         } else {
3308                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3309                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3310                                                 a += 3;    /* sub */
3311                                                 a += 10;   /* mov_imm_reg */
3312                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3313                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3314                                         }
3315
3316                                         a += 3;    /* cmp */
3317                                         a += 6;    /* jcc */
3318
3319                                         x86_64_jcc(cd, X86_64_CC_E, a);
3320
3321                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3322                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3323 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3324                     codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3325 #endif
3326                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3327                                         if (d != REG_ITMP3) {
3328                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3329                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3330 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3331                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3332 #endif
3333                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3334
3335                                         } else {
3336                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP2);
3337                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3338                                                 x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3339                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3340 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3341                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3342 #endif
3343                                         }
3344                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3345                                         x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3346                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3347                                 }
3348
3349                         } else
3350                                 panic("internal error: no inlined array checkcast");
3351                         }
3352                         M_INTMOVE(s1, d);
3353                         store_reg_to_var_int(iptr->dst, d);
3354                         break;
3355
3356                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3357
3358                         if (src->flags & INMEMORY) {
3359                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3360
3361                         } else {
3362                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3363                         }
3364                         x86_64_jcc(cd, X86_64_CC_L, 0);
3365                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3366                         break;
3367
3368                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3369
3370                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3371                         x86_64_jcc(cd, X86_64_CC_E, 0);
3372                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3373                         break;
3374
3375                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3376                                          /* op1 = dimension, val.a = array descriptor */
3377
3378                         /* check for negative sizes and copy sizes to stack if necessary  */
3379
3380                         MCODECHECK((iptr->op1 << 1) + 64);
3381
3382                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3383                                 var_to_reg_int(s2, src, REG_ITMP1);
3384                                 x86_64_testl_reg_reg(cd, s2, s2);
3385                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3386                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3387
3388                                 /* copy sizes to stack (argument numbers >= INT_ARG_CNT)      */
3389
3390                                 if (src->varkind != ARGVAR) {
3391                                         x86_64_mov_reg_membase(cd, s2, REG_SP, (s1 + INT_ARG_CNT) * 8);
3392                                 }
3393                         }
3394
3395                         /* a0 = dimension count */
3396                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3397
3398                         /* a1 = arraydescriptor */
3399                         x86_64_mov_imm_reg(cd, (s8) iptr->val.a, rd->argintregs[1]);
3400
3401                         /* a2 = pointer to dimensions = stack pointer */
3402                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3403
3404                         x86_64_mov_imm_reg(cd, (s8) builtin_nmultianewarray, REG_ITMP1);
3405                         x86_64_call_reg(cd, REG_ITMP1);
3406
3407                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3408                         M_INTMOVE(REG_RESULT, s1);
3409                         store_reg_to_var_int(iptr->dst, s1);
3410                         break;
3411
3412                 default: error("Unknown pseudo command: %d", iptr->opc);
3413         } /* switch */
3414
3415         } /* for instruction */
3416
3417         /* copy values to interface registers */
3418
3419         src = bptr->outstack;
3420         len = bptr->outdepth;
3421         MCODECHECK(64 + len);
3422         while (src) {
3423                 len--;
3424                 if ((src->varkind != STACKVAR)) {
3425                         s2 = src->type;
3426                         if (IS_FLT_DBL_TYPE(s2)) {
3427                                 var_to_reg_flt(s1, src, REG_FTMP1);
3428                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3429                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3430
3431                                 } else {
3432                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3433                                 }
3434
3435                         } else {
3436                                 var_to_reg_int(s1, src, REG_ITMP1);
3437                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3438                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3439
3440                                 } else {
3441                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3442                                 }
3443                         }
3444                 }
3445                 src = src->prev;
3446         }
3447         } /* if (bptr -> flags >= BBREACHED) */
3448         } /* for basic block */
3449
3450         {
3451
3452         /* generate bound check stubs */
3453
3454         u1 *xcodeptr = NULL;
3455         branchref *bref;
3456
3457         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3458                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3459                                   bref->branchpos,
3460                                                   cd->mcodeptr - cd->mcodebase);
3461
3462                 MCODECHECK(50);
3463
3464                 /* move index register into REG_ITMP1 */
3465                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);              /* 3 bytes  */
3466
3467                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                  /* 10 bytes */
3468                 dseg_adddata(cd, cd->mcodeptr);
3469                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3470                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3471
3472                 if (xcodeptr != NULL) {
3473                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3474
3475                 } else {
3476                         xcodeptr = cd->mcodeptr;
3477
3478                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3479                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3480
3481                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3482                         x86_64_mov_imm_reg(cd, (s8) new_arrayindexoutofboundsexception, REG_ITMP3);
3483                         x86_64_call_reg(cd, REG_ITMP3);
3484
3485                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3486                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3487
3488                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3489                         x86_64_jmp_reg(cd, REG_ITMP3);
3490                 }
3491         }
3492
3493         /* generate negative array size check stubs */
3494
3495         xcodeptr = NULL;
3496
3497         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3498                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3499                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3500                                                           bref->branchpos,
3501                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3502                         continue;
3503                 }
3504
3505                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3506                                   bref->branchpos,
3507                                                   cd->mcodeptr - cd->mcodebase);
3508
3509                 MCODECHECK(50);
3510
3511                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                         /* 10 bytes */
3512                 dseg_adddata(cd, cd->mcodeptr);
3513                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3514                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);     /* 3 bytes  */
3515
3516                 if (xcodeptr != NULL) {
3517                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3518
3519                 } else {
3520                         xcodeptr = cd->mcodeptr;
3521
3522                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3523                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3524
3525                         x86_64_mov_imm_reg(cd, (s8) new_negativearraysizeexception, REG_ITMP3);
3526                         x86_64_call_reg(cd, REG_ITMP3);
3527
3528                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3529                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3530
3531                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3532                         x86_64_jmp_reg(cd, REG_ITMP3);
3533                 }
3534         }
3535
3536         /* generate cast check stubs */
3537
3538         xcodeptr = NULL;
3539
3540         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3541                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3542                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3543                                                           bref->branchpos,
3544                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3545                         continue;
3546                 }
3547
3548                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3549                                   bref->branchpos,
3550                                                   cd->mcodeptr - cd->mcodebase);
3551
3552                 MCODECHECK(50);
3553
3554                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3555                 dseg_adddata(cd, cd->mcodeptr);
3556                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);     /* 10 bytes */
3557                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3558
3559                 if (xcodeptr != NULL) {
3560                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3561
3562                 } else {
3563                         xcodeptr = cd->mcodeptr;
3564
3565                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3566                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3567
3568                         x86_64_mov_imm_reg(cd, (s8) new_classcastexception, REG_ITMP3);
3569                         x86_64_call_reg(cd, REG_ITMP3);
3570
3571                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3572                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3573
3574                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3575                         x86_64_jmp_reg(cd, REG_ITMP3);
3576                 }
3577         }
3578
3579         /* generate divide by zero check stubs */
3580
3581         xcodeptr = NULL;
3582
3583         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3584                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3585                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3586                                                           bref->branchpos,
3587                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3588                         continue;
3589                 }
3590
3591                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3592                                   bref->branchpos,
3593                                                   cd->mcodeptr - cd->mcodebase);
3594
3595                 MCODECHECK(50);
3596
3597                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3598                 dseg_adddata(cd, cd->mcodeptr);
3599                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);      /* 10 bytes */
3600                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3601
3602                 if (xcodeptr != NULL) {
3603                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3604
3605                 } else {
3606                         xcodeptr = cd->mcodeptr;
3607
3608                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3609                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3610
3611                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
3612                         x86_64_call_reg(cd, REG_ITMP3);
3613
3614                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3615                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3616
3617                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3618                         x86_64_jmp_reg(cd, REG_ITMP3);
3619                 }
3620         }
3621
3622         /* generate exception check stubs */
3623
3624         xcodeptr = NULL;
3625
3626         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3627                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3628                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3629                                                           bref->branchpos,
3630                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3631                         continue;
3632                 }
3633
3634                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3635                                   bref->branchpos,
3636                                                   cd->mcodeptr - cd->mcodebase);
3637
3638                 MCODECHECK(50);
3639
3640                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3641                 dseg_adddata(cd, cd->mcodeptr);
3642                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3643                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3644
3645                 if (xcodeptr != NULL) {
3646                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3647
3648                 } else {
3649                         xcodeptr = cd->mcodeptr;
3650
3651 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3652                         x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3653                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0);
3654                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3655                         x86_64_call_reg(cd, REG_ITMP1);
3656                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3657                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3658                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3659                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
3660                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
3661 #else
3662                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
3663                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
3664                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
3665 #endif
3666
3667                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3668                         x86_64_jmp_reg(cd, REG_ITMP3);
3669                 }
3670         }
3671
3672         /* generate null pointer check stubs */
3673
3674         xcodeptr = NULL;
3675
3676         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3677                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3678                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3679                                                           bref->branchpos,
3680                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3681                         continue;
3682                 }
3683
3684                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3685                                   bref->branchpos,
3686                                                   cd->mcodeptr - cd->mcodebase);
3687
3688                 MCODECHECK(50);
3689
3690                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3691                 dseg_adddata(cd, cd->mcodeptr);
3692                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3693                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3694
3695                 if (xcodeptr != NULL) {
3696                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3697
3698                 } else {
3699                         xcodeptr = cd->mcodeptr;
3700
3701                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3702                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3703
3704                         x86_64_mov_imm_reg(cd, (s8) new_nullpointerexception, REG_ITMP3);
3705                         x86_64_call_reg(cd, REG_ITMP3);
3706
3707                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3708                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3709
3710                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3711                         x86_64_jmp_reg(cd, REG_ITMP3);
3712                 }
3713         }
3714         }
3715
3716         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
3717 }
3718
3719
3720 /* function createcompilerstub *************************************************
3721
3722         creates a stub routine which calls the compiler
3723
3724 *******************************************************************************/
3725
3726 #define COMPSTUBSIZE 23
3727
3728 u1 *createcompilerstub(methodinfo *m)
3729 {
3730         u1 *s = CNEW(u1, COMPSTUBSIZE);     /* memory to hold the stub            */
3731         codegendata *cd;
3732         s4 dumpsize;
3733
3734         /* mark start of dump memory area */
3735
3736         dumpsize = dump_size();
3737
3738         cd = DNEW(codegendata);
3739         cd->mcodeptr = s;
3740
3741         /* code for the stub */
3742         x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1); /* pass method to compiler     */
3743         x86_64_mov_imm_reg(cd, (s8) asm_call_jit_compiler, REG_ITMP3);/* load address */
3744         x86_64_jmp_reg(cd, REG_ITMP3);      /* jump to compiler                   */
3745
3746 #if defined(STATISTICS)
3747         if (opt_stat)
3748                 count_cstub_len += COMPSTUBSIZE;
3749 #endif
3750
3751         /* release dump area */
3752
3753         dump_release(dumpsize);
3754
3755         return s;
3756 }
3757
3758
3759 /* function removecompilerstub *************************************************
3760
3761      deletes a compilerstub from memory  (simply by freeing it)
3762
3763 *******************************************************************************/
3764
3765 void removecompilerstub(u1 *stub)
3766 {
3767         CFREE(stub, COMPSTUBSIZE);
3768 }
3769
3770
3771 /* function: createnativestub **************************************************
3772
3773         creates a stub routine which calls a native method
3774
3775 *******************************************************************************/
3776
3777 /* #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3778 /* static java_objectheader **(*callgetexceptionptrptr)() = builtin_get_exceptionptrptr; */
3779 /* #endif */
3780
3781 #define NATIVESTUBSIZE 420
3782
3783 u1 *createnativestub(functionptr f, methodinfo *m)
3784 {
3785         u1 *s = CNEW(u1, NATIVESTUBSIZE);   /* memory to hold the stub            */
3786         s4 stackframesize;                  /* size of stackframe if needed       */
3787         codegendata *cd;
3788         registerdata *rd;
3789         t_inlining_globals *id;
3790         s4 dumpsize;
3791
3792         /* mark start of dump memory area */
3793
3794         dumpsize = dump_size();
3795
3796         cd = DNEW(codegendata);
3797         rd = DNEW(registerdata);
3798         id = DNEW(t_inlining_globals);
3799
3800         /* setup registers before using it */
3801
3802         inlining_setup(m, id);
3803         reg_setup(m, rd, id);
3804
3805         cd->mcodeptr = s;
3806
3807     descriptor2types(m);                /* set paramcount and paramtypes      */
3808
3809         /* if function is static, check for initialized */
3810
3811         if (m->flags & ACC_STATIC) {
3812                 /* if class isn't yet initialized, do it */
3813                 if (!m->class->initialized) {
3814                         /* call helper function which patches this code */
3815                         x86_64_mov_imm_reg(cd, (u8) m->class, REG_ITMP1);
3816                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP2);
3817                         x86_64_call_reg(cd, REG_ITMP2);
3818                 }
3819         }
3820
3821         if (runverbose) {
3822                 s4 p, l, s1;
3823
3824                 x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3825
3826                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
3827                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
3828                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
3829                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
3830                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
3831                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
3832
3833                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
3834                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
3835                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
3836                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
3837 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8); */
3838 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8); */
3839 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8); */
3840 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8); */
3841
3842                 /* show integer hex code for float arguments */
3843                 for (p = 0, l = 0; p < m->paramcount; p++) {
3844                         if (IS_FLT_DBL_TYPE(m->paramtypes[p])) {
3845                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
3846                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
3847                                 }
3848
3849                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
3850                                 l++;
3851                         }
3852                 }
3853
3854                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1);
3855                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
3856                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
3857                 x86_64_call_reg(cd, REG_ITMP1);
3858
3859                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
3860                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
3861                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
3862                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
3863                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
3864                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
3865
3866                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
3867                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
3868                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
3869                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
3870 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]); */
3871 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]); */
3872 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]); */
3873 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]); */
3874
3875                 x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3876         }
3877
3878 #if 0
3879         x86_64_alu_imm_reg(cd, X86_64_SUB, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3880
3881         /* save callee saved float registers */
3882         x86_64_movq_reg_membase(cd, XMM15, REG_SP, 0 * 8);
3883         x86_64_movq_reg_membase(cd, XMM14, REG_SP, 1 * 8);
3884         x86_64_movq_reg_membase(cd, XMM13, REG_SP, 2 * 8);
3885         x86_64_movq_reg_membase(cd, XMM12, REG_SP, 3 * 8);
3886         x86_64_movq_reg_membase(cd, XMM11, REG_SP, 4 * 8);
3887         x86_64_movq_reg_membase(cd, XMM10, REG_SP, 5 * 8);
3888 #endif
3889
3890         /* save argument registers on stack -- if we have to */
3891         if ((m->flags & ACC_STATIC && m->paramcount > (INT_ARG_CNT - 2)) || m->paramcount > (INT_ARG_CNT - 1)) {
3892                 s4 i;
3893                 s4 paramshiftcnt = (m->flags & ACC_STATIC) ? 2 : 1;
3894                 s4 stackparamcnt = (m->paramcount > INT_ARG_CNT) ? m->paramcount - INT_ARG_CNT : 0;
3895
3896                 stackframesize = stackparamcnt + paramshiftcnt;
3897
3898                 /* keep stack 16-byte aligned */
3899                 if (!(stackframesize & 0x1))
3900                         stackframesize++;
3901
3902                 x86_64_alu_imm_reg(cd, X86_64_SUB, stackframesize * 8, REG_SP);
3903
3904                 /* copy stack arguments into new stack frame -- if any */
3905                 for (i = 0; i < stackparamcnt; i++) {
3906                         x86_64_mov_membase_reg(cd, REG_SP, (stackparamcnt + 1 + i) * 8, REG_ITMP1);
3907                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, (paramshiftcnt + i) * 8);
3908                 }
3909
3910                 if (m->flags & ACC_STATIC) {
3911                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 1 * 8);
3912                         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 0 * 8);
3913
3914                 } else {
3915                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 0 * 8);
3916                 }
3917
3918         } else {
3919                 /* keep stack 16-byte aligned */
3920                 x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3921                 stackframesize = 1;
3922         }
3923
3924         if (m->flags & ACC_STATIC) {
3925                 x86_64_mov_reg_reg(cd, rd->argintregs[3], rd->argintregs[5]);
3926                 x86_64_mov_reg_reg(cd, rd->argintregs[2], rd->argintregs[4]);
3927                 x86_64_mov_reg_reg(cd, rd->argintregs[1], rd->argintregs[3]);
3928                 x86_64_mov_reg_reg(cd, rd->argintregs[0], rd->argintregs[2]);
3929
3930                 /* put class into second argument register */
3931                 x86_64_mov_imm_reg(cd, (u8) m->class, rd->argintregs[1]);
3932
3933         } else {
3934                 x86_64_mov_reg_reg(cd, rd->argintregs[4], rd->argintregs[5]);
3935                 x86_64_mov_reg_reg(cd, rd->argintregs[3], rd->argintregs[4]);
3936                 x86_64_mov_reg_reg(cd, rd->argintregs[2], rd->argintregs[3]);
3937                 x86_64_mov_reg_reg(cd, rd->argintregs[1], rd->argintregs[2]);
3938                 x86_64_mov_reg_reg(cd, rd->argintregs[0], rd->argintregs[1]);
3939         }
3940
3941         /* put env into first argument register */
3942         x86_64_mov_imm_reg(cd, (u8) &env, rd->argintregs[0]);
3943
3944         x86_64_mov_imm_reg(cd, (u8) f, REG_ITMP1);
3945         x86_64_call_reg(cd, REG_ITMP1);
3946
3947         /* remove stackframe if there is one */
3948         if (stackframesize) {
3949                 x86_64_alu_imm_reg(cd, X86_64_ADD, stackframesize * 8, REG_SP);
3950         }
3951
3952         if (runverbose) {
3953                 x86_64_alu_imm_reg(cd, X86_64_SUB, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
3954
3955                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
3956                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
3957
3958                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
3959                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
3960                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
3961                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
3962
3963                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
3964                 x86_64_call_reg(cd, REG_ITMP1);
3965
3966                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
3967                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
3968
3969                 x86_64_alu_imm_reg(cd, X86_64_ADD, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
3970         }
3971
3972 #if 0
3973         /* restore callee saved registers */
3974         x86_64_movq_membase_reg(cd, REG_SP, 0 * 8, XMM15);
3975         x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, XMM14);
3976         x86_64_movq_membase_reg(cd, REG_SP, 2 * 8, XMM13);
3977         x86_64_movq_membase_reg(cd, REG_SP, 3 * 8, XMM12);
3978         x86_64_movq_membase_reg(cd, REG_SP, 4 * 8, XMM11);
3979         x86_64_movq_membase_reg(cd, REG_SP, 5 * 8, XMM10);
3980
3981         x86_64_alu_imm_reg(cd, X86_64_ADD, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3982 #endif
3983
3984 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3985         x86_64_push_reg(cd, REG_RESULT);
3986 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
3987         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
3988         x86_64_call_reg(cd, REG_ITMP3);
3989         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3990         x86_64_pop_reg(cd, REG_RESULT);
3991 #else
3992         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
3993         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
3994 #endif
3995         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3996         x86_64_jcc(cd, X86_64_CC_NE, 1);
3997
3998         x86_64_ret(cd);
3999
4000 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4001         x86_64_push_reg(cd, REG_ITMP3);
4002 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4003         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4004         x86_64_call_reg(cd, REG_ITMP3);
4005         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4006         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
4007 #else
4008         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4009         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
4010         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4011         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4012 #endif
4013
4014         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
4015         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
4016
4017         x86_64_mov_imm_reg(cd, (s8) asm_handle_nat_exception, REG_ITMP3);
4018         x86_64_jmp_reg(cd, REG_ITMP3);
4019
4020 #if 0
4021         {
4022                 static int stubprinted;
4023                 if (!stubprinted)
4024                         printf("stubsize: %d\n", ((long) cd->mcodeptr - (long) s));
4025                 stubprinted = 1;
4026         }
4027 #endif
4028
4029 #if defined(STATISTICS)
4030         if (opt_stat)
4031                 count_nstub_len += NATIVESTUBSIZE;
4032 #endif
4033
4034         /* release dump area */
4035
4036         dump_release(dumpsize);
4037
4038         return s;
4039 }
4040
4041
4042 /* function: removenativestub **************************************************
4043
4044     removes a previously created native-stub from memory
4045
4046 *******************************************************************************/
4047
4048 void removenativestub(u1 *stub)
4049 {
4050         CFREE(stub, NATIVESTUBSIZE);
4051 }
4052
4053
4054 /*
4055  * These are local overrides for various environment variables in Emacs.
4056  * Please do not remove this and leave it at the end of the file, where
4057  * Emacs will automagically detect them.
4058  * ---------------------------------------------------------------------
4059  * Local variables:
4060  * mode: c
4061  * indent-tabs-mode: t
4062  * c-basic-offset: 4
4063  * tab-width: 4
4064  * End:
4065  */