src/vm/jit/x86_64/codegen.c

   1 /* jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003
   4    Institut f. Computersprachen, TU Wien
   5    R. Grafl, A. Krall, C. Kruegel, C. Oates, R. Obermaisser, M. Probst,
   6    S. Ring, E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich,
   7    J. Wenninger
   8
   9    This file is part of CACAO.
  10
  11    This program is free software; you can redistribute it and/or
  12    modify it under the terms of the GNU General Public License as
  13    published by the Free Software Foundation; either version 2, or (at
  14    your option) any later version.
  15
  16    This program is distributed in the hope that it will be useful, but
  17    WITHOUT ANY WARRANTY; without even the implied warranty of
  18    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19    General Public License for more details.
  20
  21    You should have received a copy of the GNU General Public License
  22    along with this program; if not, write to the Free Software
  23    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  24    02111-1307, USA.
  25
  26    Contact: cacao@complang.tuwien.ac.at
  27
  28    Authors: Andreas Krall
  29             Christian Thalinger
  30
  31    $Id: codegen.c 1451 2004-11-05 14:14:15Z twisti $
  32
  33 */
  34
  35 #define _GNU_SOURCE
  36
  37 #include "global.h"
  38 #include <stdio.h>
  39 #include <signal.h>
  40 #include <sys/ucontext.h>
  41 #include "builtin.h"
  42 #include "asmpart.h"
  43 #include "jni.h"
  44 #include "loader.h"
  45 #include "tables.h"
  46 #include "native.h"
  47 #include "jit/jit.h"
  48 #include "jit/reg.h"
  49 #include "jit/parse.h"
  50 #include "jit/x86_64/codegen.h"
  51 #include "jit/x86_64/emitfuncs.h"
  52 #include "jit/x86_64/types.h"
  53
  54
  55 /* register descripton - array ************************************************/
  56
  57 /* #define REG_RES   0         reserved register for OS or code generator     */
  58 /* #define REG_RET   1         return value register                          */
  59 /* #define REG_EXC   2         exception value register (only old jit)        */
  60 /* #define REG_SAV   3         (callee) saved register                        */
  61 /* #define REG_TMP   4         scratch temporary register (caller saved)      */
  62 /* #define REG_ARG   5         argument register (caller saved)               */
  63
  64 /* #define REG_END   -1        last entry in tables                           */
  65
  66 int nregdescint[] = {
  67     REG_RET, REG_ARG, REG_ARG, REG_TMP, REG_RES, REG_SAV, REG_ARG, REG_ARG,
  68     REG_ARG, REG_ARG, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV,
  69     REG_END
  70 };
  71
  72
  73 int nregdescfloat[] = {
  74         /*      REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP, */
  75         /*      REG_RES, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV, REG_SAV, */
  76     REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  77     REG_RES, REG_RES, REG_RES, REG_TMP, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  78     REG_END
  79 };
  80
  81
  82 /* include independent code generation stuff -- include after register        */
  83 /* descriptions to avoid extern definitions                                   */
  84
  85 #include "jit/codegen.inc"
  86 #include "jit/reg.inc"
  87
  88
  89 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
  90 void thread_restartcriticalsection(ucontext_t *uc)
  91 {
  92         void *critical;
  93
  94         critical = thread_checkcritical((void *) uc->uc_mcontext.gregs[REG_RIP]);
  95
  96         if (critical)
  97                 uc->uc_mcontext.gregs[REG_RIP] = (u8) critical;
  98 }
  99 #endif
 100
 101
 102 /* NullPointerException signal handler for hardware null pointer check */
 103
 104 void catch_NullPointerException(int sig, siginfo_t *siginfo, void *_p)
 105 {
 106         sigset_t nsig;
 107         /*      int      instr; */
 108         /*      long     faultaddr; */
 109
 110         struct ucontext *_uc = (struct ucontext *) _p;
 111         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 112         struct sigaction act;
 113         java_objectheader *xptr;
 114
 115         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 116
 117
 118 /*      instr = *((int*)(sigctx->rip)); */
 119 /*      faultaddr = sigctx->sc_regs[(instr >> 16) & 0x1f]; */
 120
 121 /*      if (faultaddr == 0) { */
 122         act.sa_sigaction = (void *) catch_NullPointerException; /* reinstall handler */
 123         act.sa_flags = SA_SIGINFO;
 124         sigaction(sig, &act, NULL);
 125
 126         sigemptyset(&nsig);
 127         sigaddset(&nsig, sig);
 128         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 129
 130         xptr = new_nullpointerexception();
 131
 132         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 133         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 134         sigctx->rip = (u8) asm_handle_exception;
 135
 136         return;
 137
 138 /*      } else { */
 139 /*              faultaddr += (long) ((instr << 16) >> 16); */
 140 /*              fprintf(stderr, "faulting address: 0x%08x\n", faultaddr); */
 141 /*              panic("Stack overflow"); */
 142 /*      } */
 143 }
 144
 145
 146 /* ArithmeticException signal handler for hardware divide by zero check */
 147
 148 void catch_ArithmeticException(int sig, siginfo_t *siginfo, void *_p)
 149 {
 150         sigset_t nsig;
 151
 152         struct ucontext *_uc = (struct ucontext *) _p;
 153         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 154         struct sigaction act;
 155         java_objectheader *xptr;
 156
 157         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 158
 159         act.sa_sigaction = (void *) catch_ArithmeticException; /* reinstall handler */
 160         act.sa_flags = SA_SIGINFO;
 161         sigaction(sig, &act, NULL);
 162
 163         sigemptyset(&nsig);
 164         sigaddset(&nsig, sig);
 165         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 166
 167         xptr = new_arithmeticexception();
 168
 169         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 170         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 171         sigctx->rip = (u8) asm_handle_exception;
 172
 173         return;
 174 }
 175
 176
 177 void init_exceptions(void)
 178 {
 179         struct sigaction act;
 180
 181         /* install signal handlers we need to convert to exceptions */
 182         sigemptyset(&act.sa_mask);
 183
 184         if (!checknull) {
 185 #if defined(SIGSEGV)
 186                 act.sa_sigaction = (void *) catch_NullPointerException;
 187                 act.sa_flags = SA_SIGINFO;
 188                 sigaction(SIGSEGV, &act, NULL);
 189 #endif
 190
 191 #if defined(SIGBUS)
 192                 act.sa_sigaction = (void *) catch_NullPointerException;
 193                 act.sa_flags = SA_SIGINFO;
 194                 sigaction(SIGBUS, &act, NULL);
 195 #endif
 196         }
 197
 198         act.sa_sigaction = (void *) catch_ArithmeticException;
 199         act.sa_flags = SA_SIGINFO;
 200         sigaction(SIGFPE, &act, NULL);
 201 }
 202
 203
 204 /* function gen_mcode **********************************************************
 205
 206         generates machine code
 207
 208 *******************************************************************************/
 209
 210 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
 211 {
 212         s4 len, s1, s2, s3, d;
 213         s8 a;
 214         s4 parentargs_base;
 215         stackptr        src;
 216         varinfo        *var;
 217         basicblock     *bptr;
 218         instruction    *iptr;
 219         exceptiontable *ex;
 220
 221         {
 222         s4 i, p, pa, t, l;
 223         s4 savedregs_num;
 224
 225         savedregs_num = 0;
 226
 227         /* space to save used callee saved registers */
 228
 229         savedregs_num += (rd->savintregcnt - rd->maxsavintreguse);
 230         savedregs_num += (rd->savfltregcnt - rd->maxsavfltreguse);
 231
 232         parentargs_base = rd->maxmemuse + savedregs_num;
 233
 234 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 235
 236         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 237                 parentargs_base++;
 238
 239 #endif
 240
 241     /* keep stack 16-byte aligned for calls into native code e.g. libc or jni */
 242     /* (alignment problems with movaps)                                       */
 243
 244         if (!(parentargs_base & 0x1)) {
 245                 parentargs_base++;
 246         }
 247
 248         /* create method header */
 249
 250         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 251         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 252
 253 #if defined(USE_THREADS)
 254
 255         /* IsSync contains the offset relative to the stack pointer for the
 256            argument of monitor_exit used in the exception handler. Since the
 257            offset could be zero and give a wrong meaning of the flag it is
 258            offset by one.
 259         */
 260
 261         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 262                 (void) dseg_adds4(cd, (rd->maxmemuse + 1) * 8);     /* IsSync         */
 263         else
 264
 265 #endif
 266
 267                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 268
 269         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 270         (void) dseg_adds4(cd, rd->savintregcnt - rd->maxsavintreguse);/* IntSave  */
 271         (void) dseg_adds4(cd, rd->savfltregcnt - rd->maxsavfltreguse);/* FltSave  */
 272         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 273
 274         /* create exception table */
 275
 276         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 277                 dseg_addtarget(cd, ex->start);
 278                 dseg_addtarget(cd, ex->end);
 279                 dseg_addtarget(cd, ex->handler);
 280                 (void) dseg_addaddress(cd, ex->catchtype);
 281         }
 282
 283         /* initialize mcode variables */
 284
 285         cd->mcodeptr = (u1 *) cd->mcodebase;
 286         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 287         MCODECHECK(128 + m->paramcount);
 288
 289         /* create stack frame (if necessary) */
 290
 291         if (parentargs_base) {
 292                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 293         }
 294
 295         /* save return address and used callee saved registers */
 296
 297         p = parentargs_base;
 298         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
 299                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 300         }
 301         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
 302                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 303         }
 304
 305         /* save monitorenter argument */
 306
 307 #if defined(USE_THREADS)
 308         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 309                 if (m->flags & ACC_STATIC) {
 310                         x86_64_mov_imm_reg(cd, (s8) m->class, REG_ITMP1);
 311                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, rd->maxmemuse * 8);
 312
 313                 } else {
 314                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, rd->maxmemuse * 8);
 315                 }
 316         }
 317 #endif
 318
 319         /* copy argument registers to stack and call trace function with pointer
 320            to arguments on stack.
 321         */
 322         if (runverbose) {
 323                 x86_64_alu_imm_reg(cd, X86_64_SUB, (6 + 8 + 1 + 1) * 8, REG_SP);
 324
 325                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
 326                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
 327                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
 328                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
 329                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
 330                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
 331
 332                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
 333                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
 334                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
 335                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
 336 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8); */
 337 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8); */
 338 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8); */
 339 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8); */
 340
 341                 for (p = 0, l = 0; p < m->paramcount; p++) {
 342                         t = m->paramtypes[p];
 343
 344                         if (IS_FLT_DBL_TYPE(t)) {
 345                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
 346                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
 347                                 }
 348
 349                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 350                                 l++;
 351                         }
 352                 }
 353
 354                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP2);
 355                 x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 356                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
 357                 x86_64_call_reg(cd, REG_ITMP1);
 358
 359                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
 360                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
 361                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
 362                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
 363                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
 364                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
 365
 366                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
 367                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
 368                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
 369                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
 370 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]); */
 371 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]); */
 372 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]); */
 373 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]); */
 374
 375                 x86_64_alu_imm_reg(cd, X86_64_ADD, (6 + 8 + 1 + 1) * 8, REG_SP);
 376         }
 377
 378         /* take arguments out of register or stack frame */
 379
 380         for (p = 0, l = 0, s1 = 0, s2 = 0; p < m->paramcount; p++) {
 381                 t = m->paramtypes[p];
 382                 var = &(rd->locals[l][t]);
 383                 l++;
 384                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 385                         l++;
 386                 if (var->type < 0) {
 387                         if (IS_INT_LNG_TYPE(t)) {
 388                                 s1++;
 389                         } else {
 390                                 s2++;
 391                         }
 392                         continue;
 393                 }
 394                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 395                         if (s1 < INT_ARG_CNT) {                  /* register arguments    */
 396                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 397                                         M_INTMOVE(rd->argintregs[s1], var->regoff);
 398
 399                                 } else {                             /* reg arg -> spilled    */
 400                                     x86_64_mov_reg_membase(cd, rd->argintregs[s1], REG_SP, var->regoff * 8);
 401                                 }
 402
 403                         } else {                                 /* stack arguments       */
 404                                 pa = s1 - INT_ARG_CNT;
 405                                 if (s2 >= FLT_ARG_CNT) {
 406                                         pa += s2 - FLT_ARG_CNT;
 407                                 }
 408                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 409                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);    /* + 8 for return address */
 410                                 } else {                             /* stack arg -> spilled  */
 411                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_ITMP1);    /* + 8 for return address */
 412                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, var->regoff * 8);
 413                                 }
 414                         }
 415                         s1++;
 416
 417                 } else {                                     /* floating args         */
 418                         if (s2 < FLT_ARG_CNT) {                /* register arguments    */
 419                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 420                                         M_FLTMOVE(rd->argfltregs[s2], var->regoff);
 421
 422                                 } else {                                         /* reg arg -> spilled    */
 423                                         x86_64_movq_reg_membase(cd, rd->argfltregs[s2], REG_SP, var->regoff * 8);
 424                                 }
 425
 426                         } else {                                 /* stack arguments       */
 427                                 pa = s2 - FLT_ARG_CNT;
 428                                 if (s1 >= INT_ARG_CNT) {
 429                                         pa += s1 - INT_ARG_CNT;
 430                                 }
 431                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 432                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);
 433
 434                                 } else {
 435                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_FTMP1);
 436                                         x86_64_movq_reg_membase(cd, REG_FTMP1, REG_SP, var->regoff * 8);
 437                                 }
 438                         }
 439                         s2++;
 440                 }
 441         }  /* end for */
 442
 443         /* call monitorenter function */
 444
 445 #if defined(USE_THREADS)
 446         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 447                 s8 func_enter = (m->flags & ACC_STATIC) ?
 448                         (s8) builtin_staticmonitorenter : (s8) builtin_monitorenter;
 449                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
 450                 x86_64_mov_imm_reg(cd, func_enter, REG_ITMP1);
 451                 x86_64_call_reg(cd, REG_ITMP1);
 452         }
 453 #endif
 454         }
 455
 456         /* end of header generation */
 457
 458         /* walk through all basic blocks */
 459         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 460
 461                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 462
 463                 if (bptr->flags >= BBREACHED) {
 464
 465                         /* branch resolving */
 466
 467                         branchref *bref;
 468                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 469                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 470                                                                   bref->branchpos,
 471                                                                   bptr->mpc);
 472                         }
 473
 474                 /* copy interface registers to their destination */
 475
 476                 src = bptr->instack;
 477                 len = bptr->indepth;
 478                 MCODECHECK(64 + len);
 479                 while (src != NULL) {
 480                         len--;
 481                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 482                                 if (bptr->type == BBTYPE_SBR) {
 483                                         d = reg_of_var(rd, src, REG_ITMP1);
 484                                         x86_64_pop_reg(cd, d);
 485                                         store_reg_to_var_int(src, d);
 486
 487                                 } else if (bptr->type == BBTYPE_EXH) {
 488                                         d = reg_of_var(rd, src, REG_ITMP1);
 489                                         M_INTMOVE(REG_ITMP1, d);
 490                                         store_reg_to_var_int(src, d);
 491                                 }
 492
 493                         } else {
 494                                 d = reg_of_var(rd, src, REG_ITMP1);
 495                                 if ((src->varkind != STACKVAR)) {
 496                                         s2 = src->type;
 497                                         if (IS_FLT_DBL_TYPE(s2)) {
 498                                                 s1 = rd->interfaces[len][s2].regoff;
 499                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 500                                                         M_FLTMOVE(s1, d);
 501
 502                                                 } else {
 503                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 504                                                 }
 505                                                 store_reg_to_var_flt(src, d);
 506
 507                                         } else {
 508                                                 s1 = rd->interfaces[len][s2].regoff;
 509                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 510                                                         M_INTMOVE(s1, d);
 511
 512                                                 } else {
 513                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 514                                                 }
 515                                                 store_reg_to_var_int(src, d);
 516                                         }
 517                                 }
 518                         }
 519                         src = src->prev;
 520                 }
 521
 522                 /* walk through all instructions */
 523
 524                 src = bptr->instack;
 525                 len = bptr->icount;
 526                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 527
 528                         MCODECHECK(64);   /* an instruction usually needs < 64 words      */
 529                         switch (iptr->opc) {
 530
 531                         case ICMD_NOP:    /* ...  ==> ...                                 */
 532                                 break;
 533
 534                         case ICMD_NULLCHECKPOP: /* ..., objectref  ==> ...                */
 535                                 if (src->flags & INMEMORY) {
 536                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 537
 538                                 } else {
 539                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 540                                 }
 541                                 x86_64_jcc(cd, X86_64_CC_E, 0);
 542                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 543                                 break;
 544
 545                 /* constant operations ************************************************/
 546
 547                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 548                                       /* op1 = 0, val.i = constant                    */
 549
 550                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 551                         if (iptr->val.i == 0) {
 552                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 553                         } else {
 554                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 555                         }
 556                         store_reg_to_var_int(iptr->dst, d);
 557                         break;
 558
 559                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 560                                       /* op1 = 0, val.a = constant                    */
 561
 562                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 563                         if (iptr->val.a == 0) {
 564                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 565                         } else {
 566                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 567                         }
 568                         store_reg_to_var_int(iptr->dst, d);
 569                         break;
 570
 571                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 572                                       /* op1 = 0, val.l = constant                    */
 573
 574                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 575                         if (iptr->val.l == 0) {
 576                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 577                         } else {
 578                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 579                         }
 580                         store_reg_to_var_int(iptr->dst, d);
 581                         break;
 582
 583                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 584                                       /* op1 = 0, val.f = constant                    */
 585
 586                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 587                         a = dseg_addfloat(cd, iptr->val.f);
 588                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 589                         store_reg_to_var_flt(iptr->dst, d);
 590                         break;
 591
 592                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 593                                       /* op1 = 0, val.d = constant                    */
 594
 595                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 596                         a = dseg_adddouble(cd, iptr->val.d);
 597                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 598                         store_reg_to_var_flt(iptr->dst, d);
 599                         break;
 600
 601
 602                 /* load/store operations **********************************************/
 603
 604                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 605                                       /* op1 = local variable                         */
 606
 607                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 608                         if ((iptr->dst->varkind == LOCALVAR) &&
 609                             (iptr->dst->varnum == iptr->op1)) {
 610                                 break;
 611                         }
 612                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 613                         if (var->flags & INMEMORY) {
 614                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 615                                 store_reg_to_var_int(iptr->dst, d);
 616
 617                         } else {
 618                                 if (iptr->dst->flags & INMEMORY) {
 619                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 620
 621                                 } else {
 622                                         M_INTMOVE(var->regoff, d);
 623                                 }
 624                         }
 625                         break;
 626
 627                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 628                 case ICMD_ALOAD:      /* op1 = local variable                         */
 629
 630                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 631                         if ((iptr->dst->varkind == LOCALVAR) &&
 632                             (iptr->dst->varnum == iptr->op1)) {
 633                                 break;
 634                         }
 635                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 636                         if (var->flags & INMEMORY) {
 637                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 638                                 store_reg_to_var_int(iptr->dst, d);
 639
 640                         } else {
 641                                 if (iptr->dst->flags & INMEMORY) {
 642                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 643
 644                                 } else {
 645                                         M_INTMOVE(var->regoff, d);
 646                                 }
 647                         }
 648                         break;
 649
 650                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 651                 case ICMD_DLOAD:      /* op1 = local variable                         */
 652
 653                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 654                         if ((iptr->dst->varkind == LOCALVAR) &&
 655                             (iptr->dst->varnum == iptr->op1)) {
 656                                 break;
 657                         }
 658                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 659                         if (var->flags & INMEMORY) {
 660                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 661                                 store_reg_to_var_flt(iptr->dst, d);
 662
 663                         } else {
 664                                 if (iptr->dst->flags & INMEMORY) {
 665                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 666
 667                                 } else {
 668                                         M_FLTMOVE(var->regoff, d);
 669                                 }
 670                         }
 671                         break;
 672
 673                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 674                 case ICMD_LSTORE:     /* op1 = local variable                         */
 675                 case ICMD_ASTORE:
 676
 677                         if ((src->varkind == LOCALVAR) &&
 678                             (src->varnum == iptr->op1)) {
 679                                 break;
 680                         }
 681                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 682                         if (var->flags & INMEMORY) {
 683                                 var_to_reg_int(s1, src, REG_ITMP1);
 684                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 685
 686                         } else {
 687                                 var_to_reg_int(s1, src, var->regoff);
 688                                 M_INTMOVE(s1, var->regoff);
 689                         }
 690                         break;
 691
 692                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 693                 case ICMD_DSTORE:     /* op1 = local variable                         */
 694
 695                         if ((src->varkind == LOCALVAR) &&
 696                             (src->varnum == iptr->op1)) {
 697                                 break;
 698                         }
 699                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 700                         if (var->flags & INMEMORY) {
 701                                 var_to_reg_flt(s1, src, REG_FTMP1);
 702                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 703
 704                         } else {
 705                                 var_to_reg_flt(s1, src, var->regoff);
 706                                 M_FLTMOVE(s1, var->regoff);
 707                         }
 708                         break;
 709
 710
 711                 /* pop/dup/swap operations ********************************************/
 712
 713                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 714
 715                 case ICMD_POP:        /* ..., value  ==> ...                          */
 716                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 717                         break;
 718
 719                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 720                         M_COPY(src, iptr->dst);
 721                         break;
 722
 723                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 724
 725                         M_COPY(src,       iptr->dst);
 726                         M_COPY(src->prev, iptr->dst->prev);
 727                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 728                         break;
 729
 730                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 731
 732                         M_COPY(src,             iptr->dst);
 733                         M_COPY(src->prev,       iptr->dst->prev);
 734                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 735                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 736                         break;
 737
 738                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 739
 740                         M_COPY(src,       iptr->dst);
 741                         M_COPY(src->prev, iptr->dst->prev);
 742                         break;
 743
 744                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 745
 746                         M_COPY(src,             iptr->dst);
 747                         M_COPY(src->prev,       iptr->dst->prev);
 748                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 749                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 750                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 751                         break;
 752
 753                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 754
 755                         M_COPY(src,                   iptr->dst);
 756                         M_COPY(src->prev,             iptr->dst->prev);
 757                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 758                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 759                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 760                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 761                         break;
 762
 763                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 764
 765                         M_COPY(src,       iptr->dst->prev);
 766                         M_COPY(src->prev, iptr->dst);
 767                         break;
 768
 769
 770                 /* integer operations *************************************************/
 771
 772                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 773
 774                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 775                         if (iptr->dst->flags & INMEMORY) {
 776                                 if (src->flags & INMEMORY) {
 777                                         if (src->regoff == iptr->dst->regoff) {
 778                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 779
 780                                         } else {
 781                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 782                                                 x86_64_negl_reg(cd, REG_ITMP1);
 783                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 784                                         }
 785
 786                                 } else {
 787                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 788                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 789                                 }
 790
 791                         } else {
 792                                 if (src->flags & INMEMORY) {
 793                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 794                                         x86_64_negl_reg(cd, d);
 795
 796                                 } else {
 797                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 798                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 799                                 }
 800                         }
 801                         break;
 802
 803                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 804
 805                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 806                         if (iptr->dst->flags & INMEMORY) {
 807                                 if (src->flags & INMEMORY) {
 808                                         if (src->regoff == iptr->dst->regoff) {
 809                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 810
 811                                         } else {
 812                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 813                                                 x86_64_neg_reg(cd, REG_ITMP1);
 814                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 815                                         }
 816
 817                                 } else {
 818                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 819                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 820                                 }
 821
 822                         } else {
 823                                 if (src->flags & INMEMORY) {
 824                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 825                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 826
 827                                 } else {
 828                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 829                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 830                                 }
 831                         }
 832                         break;
 833
 834                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 835
 836                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 837                         if (src->flags & INMEMORY) {
 838                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 839
 840                         } else {
 841                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 842                         }
 843                         store_reg_to_var_int(iptr->dst, d);
 844                         break;
 845
 846                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 847
 848                         var_to_reg_int(s1, src, REG_ITMP1);
 849                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 850                         M_INTMOVE(s1, d);
 851                         store_reg_to_var_int(iptr->dst, d);
 852                         break;
 853
 854                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 855
 856                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 857                         if (src->flags & INMEMORY) {
 858                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 859
 860                         } else {
 861                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 862                         }
 863                         store_reg_to_var_int(iptr->dst, d);
 864                         break;
 865
 866                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 867
 868                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 869                         if (src->flags & INMEMORY) {
 870                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 871
 872                         } else {
 873                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 874                         }
 875                         store_reg_to_var_int(iptr->dst, d);
 876                         break;
 877
 878                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 879
 880                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 881                         if (src->flags & INMEMORY) {
 882                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 883
 884                         } else {
 885                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 886                         }
 887                         store_reg_to_var_int(iptr->dst, d);
 888                         break;
 889
 890
 891                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 892
 893                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 894                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 895                         break;
 896
 897                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 898                                       /* val.i = constant                             */
 899
 900                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 901                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 902                         break;
 903
 904                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 905
 906                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 907                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 908                         break;
 909
 910                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 911                                       /* val.l = constant                             */
 912
 913                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 914                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 915                         break;
 916
 917                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 918
 919                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 920                         if (iptr->dst->flags & INMEMORY) {
 921                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 922                                         if (src->prev->regoff == iptr->dst->regoff) {
 923                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 924                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 925
 926                                         } else {
 927                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 928                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 929                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 930                                         }
 931
 932                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 933                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 934                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 935                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 936
 937                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 938                                         if (src->prev->regoff == iptr->dst->regoff) {
 939                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 940
 941                                         } else {
 942                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 943                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 944                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 945                                         }
 946
 947                                 } else {
 948                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 949                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 950                                 }
 951
 952                         } else {
 953                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 954                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 955                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 956
 957                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 958                                         M_INTMOVE(src->prev->regoff, d);
 959                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 960
 961                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 962                                         /* workaround for reg alloc */
 963                                         if (src->regoff == iptr->dst->regoff) {
 964                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 965                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 966                                                 M_INTMOVE(REG_ITMP1, d);
 967
 968                                         } else {
 969                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 970                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 971                                         }
 972
 973                                 } else {
 974                                         /* workaround for reg alloc */
 975                                         if (src->regoff == iptr->dst->regoff) {
 976                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 977                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 978                                                 M_INTMOVE(REG_ITMP1, d);
 979
 980                                         } else {
 981                                                 M_INTMOVE(src->prev->regoff, d);
 982                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 983                                         }
 984                                 }
 985                         }
 986                         break;
 987
 988                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 989                                       /* val.i = constant                             */
 990
 991                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 992                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 993                         break;
 994
 995                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 996
 997                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 998                         if (iptr->dst->flags & INMEMORY) {
 999                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1000                                         if (src->prev->regoff == iptr->dst->regoff) {
1001                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1002                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1003
1004                                         } else {
1005                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1006                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1007                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1008                                         }
1009
1010                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1011                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1012                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1013                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1014
1015                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1016                                         if (src->prev->regoff == iptr->dst->regoff) {
1017                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1018
1019                                         } else {
1020                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1021                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1022                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1023                                         }
1024
1025                                 } else {
1026                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1027                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1028                                 }
1029
1030                         } else {
1031                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1032                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1033                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1034
1035                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1036                                         M_INTMOVE(src->prev->regoff, d);
1037                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1038
1039                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1040                                         /* workaround for reg alloc */
1041                                         if (src->regoff == iptr->dst->regoff) {
1042                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1043                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1044                                                 M_INTMOVE(REG_ITMP1, d);
1045
1046                                         } else {
1047                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1048                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1049                                         }
1050
1051                                 } else {
1052                                         /* workaround for reg alloc */
1053                                         if (src->regoff == iptr->dst->regoff) {
1054                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1055                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1056                                                 M_INTMOVE(REG_ITMP1, d);
1057
1058                                         } else {
1059                                                 M_INTMOVE(src->prev->regoff, d);
1060                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1061                                         }
1062                                 }
1063                         }
1064                         break;
1065
1066                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1067                                       /* val.l = constant                             */
1068
1069                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1070                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1071                         break;
1072
1073                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1074
1075                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1076                         if (iptr->dst->flags & INMEMORY) {
1077                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1078                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1079                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1080                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1081
1082                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1083                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1084                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1085                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1086
1087                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1088                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1089                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1090                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1091
1092                                 } else {
1093                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1094                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1095                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1096                                 }
1097
1098                         } else {
1099                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1100                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1101                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1102
1103                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1104                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1105                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1106
1107                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1108                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1109                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1110
1111                                 } else {
1112                                         if (src->regoff == iptr->dst->regoff) {
1113                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1114
1115                                         } else {
1116                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1117                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1118                                         }
1119                                 }
1120                         }
1121                         break;
1122
1123                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1124                                       /* val.i = constant                             */
1125
1126                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1127                         if (iptr->dst->flags & INMEMORY) {
1128                                 if (src->flags & INMEMORY) {
1129                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1130                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1131
1132                                 } else {
1133                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1134                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1135                                 }
1136
1137                         } else {
1138                                 if (src->flags & INMEMORY) {
1139                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1140
1141                                 } else {
1142                                         if (iptr->val.i == 2) {
1143                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1144                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1145
1146                                         } else {
1147                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1148                                         }
1149                                 }
1150                         }
1151                         break;
1152
1153                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1154
1155                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1156                         if (iptr->dst->flags & INMEMORY) {
1157                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1158                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1159                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1160                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1161
1162                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1163                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1164                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1165                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1166
1167                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1168                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1169                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1170                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1171
1172                                 } else {
1173                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1174                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1175                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1176                                 }
1177
1178                         } else {
1179                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1180                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1181                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1182
1183                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1184                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1185                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1186
1187                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1188                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1189                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1190
1191                                 } else {
1192                                         if (src->regoff == iptr->dst->regoff) {
1193                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1194
1195                                         } else {
1196                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1197                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1198                                         }
1199                                 }
1200                         }
1201                         break;
1202
1203                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1204                                       /* val.l = constant                             */
1205
1206                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1207                         if (iptr->dst->flags & INMEMORY) {
1208                                 if (src->flags & INMEMORY) {
1209                                         if (x86_64_is_imm32(iptr->val.l)) {
1210                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1211
1212                                         } else {
1213                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1214                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1215                                         }
1216                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1217
1218                                 } else {
1219                                         if (x86_64_is_imm32(iptr->val.l)) {
1220                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1221
1222                                         } else {
1223                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1224                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1225                                         }
1226                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1227                                 }
1228
1229                         } else {
1230                                 if (src->flags & INMEMORY) {
1231                                         if (x86_64_is_imm32(iptr->val.l)) {
1232                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1233
1234                                         } else {
1235                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1236                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1237                                         }
1238
1239                                 } else {
1240                                         /* should match in many cases */
1241                                         if (iptr->val.l == 2) {
1242                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1243                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1244
1245                                         } else {
1246                                                 if (x86_64_is_imm32(iptr->val.l)) {
1247                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1248
1249                                                 } else {
1250                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1251                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1252                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1253                                                 }
1254                                         }
1255                                 }
1256                         }
1257                         break;
1258
1259                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1260
1261                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1262                 if (src->prev->flags & INMEMORY) {
1263                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1264
1265                         } else {
1266                                 M_INTMOVE(src->prev->regoff, RAX);
1267                         }
1268
1269                         if (src->flags & INMEMORY) {
1270                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1271
1272                         } else {
1273                                 M_INTMOVE(src->regoff, REG_ITMP3);
1274                         }
1275                         gen_div_check(src);
1276
1277                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1278                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1279                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1280                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1281
1282                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1283                         x86_64_cltd(cd);
1284                         x86_64_idivl_reg(cd, REG_ITMP3);
1285
1286                         if (iptr->dst->flags & INMEMORY) {
1287                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1288                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1289
1290                         } else {
1291                                 M_INTMOVE(RAX, iptr->dst->regoff);
1292
1293                                 if (iptr->dst->regoff != RDX) {
1294                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1295                                 }
1296                         }
1297                         break;
1298
1299                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1300
1301                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1302                         if (src->prev->flags & INMEMORY) {
1303                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1304
1305                         } else {
1306                                 M_INTMOVE(src->prev->regoff, RAX);
1307                         }
1308
1309                         if (src->flags & INMEMORY) {
1310                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1311
1312                         } else {
1313                                 M_INTMOVE(src->regoff, REG_ITMP3);
1314                         }
1315                         gen_div_check(src);
1316
1317                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1318                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1319                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1320                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1321                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1322
1323                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1324                         x86_64_cltd(cd);
1325                         x86_64_idivl_reg(cd, REG_ITMP3);
1326
1327                         if (iptr->dst->flags & INMEMORY) {
1328                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1329                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1330
1331                         } else {
1332                                 M_INTMOVE(RDX, iptr->dst->regoff);
1333
1334                                 if (iptr->dst->regoff != RDX) {
1335                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1336                                 }
1337                         }
1338                         break;
1339
1340                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1341                                       /* val.i = constant                             */
1342
1343                         var_to_reg_int(s1, src, REG_ITMP1);
1344                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1345                         M_INTMOVE(s1, REG_ITMP1);
1346                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1347                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1348                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1349                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1350                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1351                         store_reg_to_var_int(iptr->dst, d);
1352                         break;
1353
1354                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1355                                       /* val.i = constant                             */
1356
1357                         var_to_reg_int(s1, src, REG_ITMP1);
1358                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1359                         M_INTMOVE(s1, REG_ITMP1);
1360                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1361                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1362                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1363                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1364                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1365                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1366                         store_reg_to_var_int(iptr->dst, d);
1367                         break;
1368
1369
1370                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1371
1372                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1373                 if (src->prev->flags & INMEMORY) {
1374                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1375
1376                         } else {
1377                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1378                         }
1379
1380                         if (src->flags & INMEMORY) {
1381                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1382
1383                         } else {
1384                                 M_INTMOVE(src->regoff, REG_ITMP3);
1385                         }
1386                         gen_div_check(src);
1387
1388                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1389                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1390                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1391                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1392                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1393
1394                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1395                         x86_64_cqto(cd);
1396                         x86_64_idiv_reg(cd, REG_ITMP3);
1397
1398                         if (iptr->dst->flags & INMEMORY) {
1399                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1400                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1401
1402                         } else {
1403                                 M_INTMOVE(RAX, iptr->dst->regoff);
1404
1405                                 if (iptr->dst->regoff != RDX) {
1406                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1407                                 }
1408                         }
1409                         break;
1410
1411                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1412
1413                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1414                         if (src->prev->flags & INMEMORY) {
1415                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1416
1417                         } else {
1418                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1419                         }
1420
1421                         if (src->flags & INMEMORY) {
1422                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1423
1424                         } else {
1425                                 M_INTMOVE(src->regoff, REG_ITMP3);
1426                         }
1427                         gen_div_check(src);
1428
1429                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1430                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1431                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1432                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1433                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1434                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1435
1436                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1437                         x86_64_cqto(cd);
1438                         x86_64_idiv_reg(cd, REG_ITMP3);
1439
1440                         if (iptr->dst->flags & INMEMORY) {
1441                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1442                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1443
1444                         } else {
1445                                 M_INTMOVE(RDX, iptr->dst->regoff);
1446
1447                                 if (iptr->dst->regoff != RDX) {
1448                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1449                                 }
1450                         }
1451                         break;
1452
1453                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1454                                       /* val.i = constant                             */
1455
1456                         var_to_reg_int(s1, src, REG_ITMP1);
1457                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1458                         M_INTMOVE(s1, REG_ITMP1);
1459                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1460                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1461                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1462                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1463                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1464                         store_reg_to_var_int(iptr->dst, d);
1465                         break;
1466
1467                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1468                                       /* val.l = constant                             */
1469
1470                         var_to_reg_int(s1, src, REG_ITMP1);
1471                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1472                         M_INTMOVE(s1, REG_ITMP1);
1473                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1474                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1475                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1476                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1477                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1478                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1479                         store_reg_to_var_int(iptr->dst, d);
1480                         break;
1481
1482                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1483
1484                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1485                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1486                         break;
1487
1488                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1489                                       /* val.i = constant                             */
1490
1491                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1492                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1493                         break;
1494
1495                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1496
1497                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1498                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1499                         break;
1500
1501                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1502                                       /* val.i = constant                             */
1503
1504                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1505                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1506                         break;
1507
1508                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1509
1510                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1511                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1512                         break;
1513
1514                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1515                                       /* val.i = constant                             */
1516
1517                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1518                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1519                         break;
1520
1521                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1522
1523                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1524                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1525                         break;
1526
1527         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1528                                           /* val.i = constant                             */
1529
1530                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1531                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1532                         break;
1533
1534                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1535
1536                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1537                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1538                         break;
1539
1540                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1541                                       /* val.i = constant                             */
1542
1543                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1544                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1545                         break;
1546
1547                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1548
1549                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1550                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1551                         break;
1552
1553                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1554                                       /* val.l = constant                             */
1555
1556                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1557                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1558                         break;
1559
1560                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1561
1562                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1563                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1564                         break;
1565
1566                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1567                                       /* val.i = constant                             */
1568
1569                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1570                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1571                         break;
1572
1573                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1574
1575                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1576                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1577                         break;
1578
1579                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1580                                       /* val.l = constant                             */
1581
1582                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1583                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1584                         break;
1585
1586                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1587
1588                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1589                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1590                         break;
1591
1592                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1593                                       /* val.i = constant                             */
1594
1595                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1596                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1597                         break;
1598
1599                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1600
1601                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1602                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1603                         break;
1604
1605                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1606                                       /* val.l = constant                             */
1607
1608                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1609                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1610                         break;
1611
1612                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1613
1614                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1615                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1616                         break;
1617
1618                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1619                                       /* val.i = constant                             */
1620
1621                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1622                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1623                         break;
1624
1625                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1626
1627                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1628                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1629                         break;
1630
1631                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1632                                       /* val.l = constant                             */
1633
1634                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1635                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1636                         break;
1637
1638
1639                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1640                                       /* op1 = variable, val.i = constant             */
1641
1642                         /* using inc and dec is definitely faster than add -- tested      */
1643                         /* with sieve                                                     */
1644
1645                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1646                         d = var->regoff;
1647                         if (var->flags & INMEMORY) {
1648                                 if (iptr->val.i == 1) {
1649                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1650
1651                                 } else if (iptr->val.i == -1) {
1652                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1653
1654                                 } else {
1655                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1656                                 }
1657
1658                         } else {
1659                                 if (iptr->val.i == 1) {
1660                                         x86_64_incl_reg(cd, d);
1661
1662                                 } else if (iptr->val.i == -1) {
1663                                         x86_64_decl_reg(cd, d);
1664
1665                                 } else {
1666                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1667                                 }
1668                         }
1669                         break;
1670
1671
1672                 /* floating operations ************************************************/
1673
1674                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1675
1676                         var_to_reg_flt(s1, src, REG_FTMP1);
1677                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1678                         a = dseg_adds4(cd, 0x80000000);
1679                         M_FLTMOVE(s1, d);
1680                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1681                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1682                         store_reg_to_var_flt(iptr->dst, d);
1683                         break;
1684
1685                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1686
1687                         var_to_reg_flt(s1, src, REG_FTMP1);
1688                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1689                         a = dseg_adds8(cd, 0x8000000000000000);
1690                         M_FLTMOVE(s1, d);
1691                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1692                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1693                         store_reg_to_var_flt(iptr->dst, d);
1694                         break;
1695
1696                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1697
1698                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1699                         var_to_reg_flt(s2, src, REG_FTMP2);
1700                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1701                         if (s1 == d) {
1702                                 x86_64_addss_reg_reg(cd, s2, d);
1703                         } else if (s2 == d) {
1704                                 x86_64_addss_reg_reg(cd, s1, d);
1705                         } else {
1706                                 M_FLTMOVE(s1, d);
1707                                 x86_64_addss_reg_reg(cd, s2, d);
1708                         }
1709                         store_reg_to_var_flt(iptr->dst, d);
1710                         break;
1711
1712                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1713
1714                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1715                         var_to_reg_flt(s2, src, REG_FTMP2);
1716                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1717                         if (s1 == d) {
1718                                 x86_64_addsd_reg_reg(cd, s2, d);
1719                         } else if (s2 == d) {
1720                                 x86_64_addsd_reg_reg(cd, s1, d);
1721                         } else {
1722                                 M_FLTMOVE(s1, d);
1723                                 x86_64_addsd_reg_reg(cd, s2, d);
1724                         }
1725                         store_reg_to_var_flt(iptr->dst, d);
1726                         break;
1727
1728                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1729
1730                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1731                         var_to_reg_flt(s2, src, REG_FTMP2);
1732                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1733                         if (s2 == d) {
1734                                 M_FLTMOVE(s2, REG_FTMP2);
1735                                 s2 = REG_FTMP2;
1736                         }
1737                         M_FLTMOVE(s1, d);
1738                         x86_64_subss_reg_reg(cd, s2, d);
1739                         store_reg_to_var_flt(iptr->dst, d);
1740                         break;
1741
1742                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1743
1744                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1745                         var_to_reg_flt(s2, src, REG_FTMP2);
1746                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1747                         if (s2 == d) {
1748                                 M_FLTMOVE(s2, REG_FTMP2);
1749                                 s2 = REG_FTMP2;
1750                         }
1751                         M_FLTMOVE(s1, d);
1752                         x86_64_subsd_reg_reg(cd, s2, d);
1753                         store_reg_to_var_flt(iptr->dst, d);
1754                         break;
1755
1756                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1757
1758                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1759                         var_to_reg_flt(s2, src, REG_FTMP2);
1760                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1761                         if (s1 == d) {
1762                                 x86_64_mulss_reg_reg(cd, s2, d);
1763                         } else if (s2 == d) {
1764                                 x86_64_mulss_reg_reg(cd, s1, d);
1765                         } else {
1766                                 M_FLTMOVE(s1, d);
1767                                 x86_64_mulss_reg_reg(cd, s2, d);
1768                         }
1769                         store_reg_to_var_flt(iptr->dst, d);
1770                         break;
1771
1772                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1773
1774                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1775                         var_to_reg_flt(s2, src, REG_FTMP2);
1776                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1777                         if (s1 == d) {
1778                                 x86_64_mulsd_reg_reg(cd, s2, d);
1779                         } else if (s2 == d) {
1780                                 x86_64_mulsd_reg_reg(cd, s1, d);
1781                         } else {
1782                                 M_FLTMOVE(s1, d);
1783                                 x86_64_mulsd_reg_reg(cd, s2, d);
1784                         }
1785                         store_reg_to_var_flt(iptr->dst, d);
1786                         break;
1787
1788                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1789
1790                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1791                         var_to_reg_flt(s2, src, REG_FTMP2);
1792                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1793                         if (s2 == d) {
1794                                 M_FLTMOVE(s2, REG_FTMP2);
1795                                 s2 = REG_FTMP2;
1796                         }
1797                         M_FLTMOVE(s1, d);
1798                         x86_64_divss_reg_reg(cd, s2, d);
1799                         store_reg_to_var_flt(iptr->dst, d);
1800                         break;
1801
1802                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1803
1804                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1805                         var_to_reg_flt(s2, src, REG_FTMP2);
1806                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1807                         if (s2 == d) {
1808                                 M_FLTMOVE(s2, REG_FTMP2);
1809                                 s2 = REG_FTMP2;
1810                         }
1811                         M_FLTMOVE(s1, d);
1812                         x86_64_divsd_reg_reg(cd, s2, d);
1813                         store_reg_to_var_flt(iptr->dst, d);
1814                         break;
1815
1816                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1817
1818                         var_to_reg_int(s1, src, REG_ITMP1);
1819                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1820                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1821                         store_reg_to_var_flt(iptr->dst, d);
1822                         break;
1823
1824                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1825
1826                         var_to_reg_int(s1, src, REG_ITMP1);
1827                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1828                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1829                         store_reg_to_var_flt(iptr->dst, d);
1830                         break;
1831
1832                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1833
1834                         var_to_reg_int(s1, src, REG_ITMP1);
1835                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1836                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1837                         store_reg_to_var_flt(iptr->dst, d);
1838                         break;
1839
1840                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1841
1842                         var_to_reg_int(s1, src, REG_ITMP1);
1843                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1844                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1845                         store_reg_to_var_flt(iptr->dst, d);
1846                         break;
1847
1848                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1849
1850                         var_to_reg_flt(s1, src, REG_FTMP1);
1851                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1852                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1853                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1854                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1855                         x86_64_jcc(cd, X86_64_CC_NE, a);
1856                         M_FLTMOVE(s1, REG_FTMP1);
1857                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1858                         x86_64_call_reg(cd, REG_ITMP2);
1859                         M_INTMOVE(REG_RESULT, d);
1860                         store_reg_to_var_int(iptr->dst, d);
1861                         break;
1862
1863                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1864
1865                         var_to_reg_flt(s1, src, REG_FTMP1);
1866                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1867                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1868                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1869                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1870                         x86_64_jcc(cd, X86_64_CC_NE, a);
1871                         M_FLTMOVE(s1, REG_FTMP1);
1872                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1873                         x86_64_call_reg(cd, REG_ITMP2);
1874                         M_INTMOVE(REG_RESULT, d);
1875                         store_reg_to_var_int(iptr->dst, d);
1876                         break;
1877
1878                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1879
1880                         var_to_reg_flt(s1, src, REG_FTMP1);
1881                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1882                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1883                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1884                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1885                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1886                         x86_64_jcc(cd, X86_64_CC_NE, a);
1887                         M_FLTMOVE(s1, REG_FTMP1);
1888                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1889                         x86_64_call_reg(cd, REG_ITMP2);
1890                         M_INTMOVE(REG_RESULT, d);
1891                         store_reg_to_var_int(iptr->dst, d);
1892                         break;
1893
1894                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1895
1896                         var_to_reg_flt(s1, src, REG_FTMP1);
1897                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1898                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1899                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1900                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1901                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1902                         x86_64_jcc(cd, X86_64_CC_NE, a);
1903                         M_FLTMOVE(s1, REG_FTMP1);
1904                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1905                         x86_64_call_reg(cd, REG_ITMP2);
1906                         M_INTMOVE(REG_RESULT, d);
1907                         store_reg_to_var_int(iptr->dst, d);
1908                         break;
1909
1910                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1911
1912                         var_to_reg_flt(s1, src, REG_FTMP1);
1913                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1914                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1915                         store_reg_to_var_flt(iptr->dst, d);
1916                         break;
1917
1918                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1919
1920                         var_to_reg_flt(s1, src, REG_FTMP1);
1921                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1922                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1923                         store_reg_to_var_flt(iptr->dst, d);
1924                         break;
1925
1926                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1927                                           /* == => 0, < => 1, > => -1 */
1928
1929                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1930                         var_to_reg_flt(s2, src, REG_FTMP2);
1931                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1932                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1933                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1934                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1935                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1936                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1937                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1938                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1939                         store_reg_to_var_int(iptr->dst, d);
1940                         break;
1941
1942                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1943                                           /* == => 0, < => 1, > => -1 */
1944
1945                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1946                         var_to_reg_flt(s2, src, REG_FTMP2);
1947                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1948                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1949                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1950                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1951                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1952                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1953                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1954                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1955                         store_reg_to_var_int(iptr->dst, d);
1956                         break;
1957
1958                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1959                                           /* == => 0, < => 1, > => -1 */
1960
1961                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1962                         var_to_reg_flt(s2, src, REG_FTMP2);
1963                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1964                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1965                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1966                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1967                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1968                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1969                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1970                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1971                         store_reg_to_var_int(iptr->dst, d);
1972                         break;
1973
1974                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1975                                           /* == => 0, < => 1, > => -1 */
1976
1977                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1978                         var_to_reg_flt(s2, src, REG_FTMP2);
1979                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1980                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1981                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1982                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1983                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1984                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1985                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1986                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1987                         store_reg_to_var_int(iptr->dst, d);
1988                         break;
1989
1990
1991                 /* memory operations **************************************************/
1992
1993                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1994
1995                         var_to_reg_int(s1, src, REG_ITMP1);
1996                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1997                         gen_nullptr_check(s1);
1998                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1999                         store_reg_to_var_int(iptr->dst, d);
2000                         break;
2001
2002                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2003
2004                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2005                         var_to_reg_int(s2, src, REG_ITMP2);
2006                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2007                         if (iptr->op1 == 0) {
2008                                 gen_nullptr_check(s1);
2009                                 gen_bound_check;
2010                         }
2011                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2012                         store_reg_to_var_int(iptr->dst, d);
2013                         break;
2014
2015                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2016
2017                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2018                         var_to_reg_int(s2, src, REG_ITMP2);
2019                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2020                         if (iptr->op1 == 0) {
2021                                 gen_nullptr_check(s1);
2022                                 gen_bound_check;
2023                         }
2024                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2025                         store_reg_to_var_int(iptr->dst, d);
2026                         break;
2027
2028                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2029
2030                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2031                         var_to_reg_int(s2, src, REG_ITMP2);
2032                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2033                         if (iptr->op1 == 0) {
2034                                 gen_nullptr_check(s1);
2035                                 gen_bound_check;
2036                         }
2037                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2038                         store_reg_to_var_int(iptr->dst, d);
2039                         break;
2040
2041                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2042
2043                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2044                         var_to_reg_int(s2, src, REG_ITMP2);
2045                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2046                         if (iptr->op1 == 0) {
2047                                 gen_nullptr_check(s1);
2048                                 gen_bound_check;
2049                         }
2050                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2051                         store_reg_to_var_flt(iptr->dst, d);
2052                         break;
2053
2054                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2055
2056                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2057                         var_to_reg_int(s2, src, REG_ITMP2);
2058                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2059                         if (iptr->op1 == 0) {
2060                                 gen_nullptr_check(s1);
2061                                 gen_bound_check;
2062                         }
2063                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2064                         store_reg_to_var_flt(iptr->dst, d);
2065                         break;
2066
2067                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2068
2069                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2070                         var_to_reg_int(s2, src, REG_ITMP2);
2071                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2072                         if (iptr->op1 == 0) {
2073                                 gen_nullptr_check(s1);
2074                                 gen_bound_check;
2075                         }
2076                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2077                         store_reg_to_var_int(iptr->dst, d);
2078                         break;
2079
2080                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2081
2082                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2083                         var_to_reg_int(s2, src, REG_ITMP2);
2084                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2085                         if (iptr->op1 == 0) {
2086                                 gen_nullptr_check(s1);
2087                                 gen_bound_check;
2088                         }
2089                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2090                         store_reg_to_var_int(iptr->dst, d);
2091                         break;
2092
2093                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2094
2095                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2096                         var_to_reg_int(s2, src, REG_ITMP2);
2097                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2098                         if (iptr->op1 == 0) {
2099                                 gen_nullptr_check(s1);
2100                                 gen_bound_check;
2101                         }
2102                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2103                         store_reg_to_var_int(iptr->dst, d);
2104                         break;
2105
2106
2107                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2108
2109                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2110                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2111                         if (iptr->op1 == 0) {
2112                                 gen_nullptr_check(s1);
2113                                 gen_bound_check;
2114                         }
2115                         var_to_reg_int(s3, src, REG_ITMP3);
2116                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2117                         break;
2118
2119                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2120
2121                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2122                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2123                         if (iptr->op1 == 0) {
2124                                 gen_nullptr_check(s1);
2125                                 gen_bound_check;
2126                         }
2127                         var_to_reg_int(s3, src, REG_ITMP3);
2128                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2129                         break;
2130
2131                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2132
2133                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2134                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2135                         if (iptr->op1 == 0) {
2136                                 gen_nullptr_check(s1);
2137                                 gen_bound_check;
2138                         }
2139                         var_to_reg_int(s3, src, REG_ITMP3);
2140                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2141                         break;
2142
2143                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2144
2145                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2146                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2147                         if (iptr->op1 == 0) {
2148                                 gen_nullptr_check(s1);
2149                                 gen_bound_check;
2150                         }
2151                         var_to_reg_flt(s3, src, REG_FTMP3);
2152                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2153                         break;
2154
2155                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2156
2157                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2158                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2159                         if (iptr->op1 == 0) {
2160                                 gen_nullptr_check(s1);
2161                                 gen_bound_check;
2162                         }
2163                         var_to_reg_flt(s3, src, REG_FTMP3);
2164                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2165                         break;
2166
2167                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2168
2169                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2170                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2171                         if (iptr->op1 == 0) {
2172                                 gen_nullptr_check(s1);
2173                                 gen_bound_check;
2174                         }
2175                         var_to_reg_int(s3, src, REG_ITMP3);
2176                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2177                         break;
2178
2179                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2180
2181                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2182                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2183                         if (iptr->op1 == 0) {
2184                                 gen_nullptr_check(s1);
2185                                 gen_bound_check;
2186                         }
2187                         var_to_reg_int(s3, src, REG_ITMP3);
2188                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2189                         break;
2190
2191                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2192
2193                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2194                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2195                         if (iptr->op1 == 0) {
2196                                 gen_nullptr_check(s1);
2197                                 gen_bound_check;
2198                         }
2199                         var_to_reg_int(s3, src, REG_ITMP3);
2200                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2201                         break;
2202
2203                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2204
2205                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2206                         var_to_reg_int(s2, src, REG_ITMP2);
2207                         if (iptr->op1 == 0) {
2208                                 gen_nullptr_check(s1);
2209                                 gen_bound_check;
2210                         }
2211                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2212                         break;
2213
2214                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2215
2216                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2217                         var_to_reg_int(s2, src, REG_ITMP2);
2218                         if (iptr->op1 == 0) {
2219                                 gen_nullptr_check(s1);
2220                                 gen_bound_check;
2221                         }
2222
2223                         if (x86_64_is_imm32(iptr->val.l)) {
2224                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2225
2226                         } else {
2227                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2228                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2229                         }
2230                         break;
2231
2232                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2233
2234                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2235                         var_to_reg_int(s2, src, REG_ITMP2);
2236                         if (iptr->op1 == 0) {
2237                                 gen_nullptr_check(s1);
2238                                 gen_bound_check;
2239                         }
2240                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2241                         break;
2242
2243                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2244
2245                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2246                         var_to_reg_int(s2, src, REG_ITMP2);
2247                         if (iptr->op1 == 0) {
2248                                 gen_nullptr_check(s1);
2249                                 gen_bound_check;
2250                         }
2251                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2252                         break;
2253
2254                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2255
2256                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2257                         var_to_reg_int(s2, src, REG_ITMP2);
2258                         if (iptr->op1 == 0) {
2259                                 gen_nullptr_check(s1);
2260                                 gen_bound_check;
2261                         }
2262                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2263                         break;
2264
2265                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2266
2267                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2268                         var_to_reg_int(s2, src, REG_ITMP2);
2269                         if (iptr->op1 == 0) {
2270                                 gen_nullptr_check(s1);
2271                                 gen_bound_check;
2272                         }
2273                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2274                         break;
2275
2276
2277                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2278                                       /* op1 = type, val.a = field address            */
2279
2280                         /* if class isn't yet initialized, do it */
2281                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2282                                 /* call helper function which patches this code */
2283                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2284                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2285                                 x86_64_call_reg(cd, REG_ITMP2);
2286                         }
2287
2288                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2289                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2290                         switch (iptr->op1) {
2291                         case TYPE_INT:
2292                                 var_to_reg_int(s2, src, REG_ITMP1);
2293                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2294                                 break;
2295                         case TYPE_LNG:
2296                         case TYPE_ADR:
2297                                 var_to_reg_int(s2, src, REG_ITMP1);
2298                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2299                                 break;
2300                         case TYPE_FLT:
2301                                 var_to_reg_flt(s2, src, REG_FTMP1);
2302                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2303                                 break;
2304                         case TYPE_DBL:
2305                                 var_to_reg_flt(s2, src, REG_FTMP1);
2306                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2307                                 break;
2308                         default: panic("internal error");
2309                         }
2310                         break;
2311
2312                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2313                                       /* op1 = type, val.a = field address            */
2314
2315                         /* if class isn't yet initialized, do it */
2316                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2317                                 /* call helper function which patches this code */
2318                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2319                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2320                                 x86_64_call_reg(cd, REG_ITMP2);
2321                         }
2322
2323                         a = dseg_addaddress(cd, &(((fieldinfo *) iptr->val.a)->value));
2324                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2325                         switch (iptr->op1) {
2326                         case TYPE_INT:
2327                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2328                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2329                                 store_reg_to_var_int(iptr->dst, d);
2330                                 break;
2331                         case TYPE_LNG:
2332                         case TYPE_ADR:
2333                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2334                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2335                                 store_reg_to_var_int(iptr->dst, d);
2336                                 break;
2337                         case TYPE_FLT:
2338                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2339                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2340                                 store_reg_to_var_flt(iptr->dst, d);
2341                                 break;
2342                         case TYPE_DBL:
2343                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2344                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2345                                 store_reg_to_var_flt(iptr->dst, d);
2346                                 break;
2347                         default: panic("internal error");
2348                         }
2349                         break;
2350
2351                 case ICMD_PUTFIELD:   /* ..., value  ==> ...                          */
2352                                       /* op1 = type, val.i = field offset             */
2353
2354                         /* if class isn't yet initialized, do it */
2355                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2356                                 /* call helper function which patches this code */
2357                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2358                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2359                                 x86_64_call_reg(cd, REG_ITMP2);
2360                         }
2361
2362                         a = ((fieldinfo *)(iptr->val.a))->offset;
2363                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2364                         switch (iptr->op1) {
2365                                 case TYPE_INT:
2366                                         var_to_reg_int(s2, src, REG_ITMP2);
2367                                         gen_nullptr_check(s1);
2368                                         x86_64_movl_reg_membase(cd, s2, s1, a);
2369                                         break;
2370                                 case TYPE_LNG:
2371                                 case TYPE_ADR:
2372                                         var_to_reg_int(s2, src, REG_ITMP2);
2373                                         gen_nullptr_check(s1);
2374                                         x86_64_mov_reg_membase(cd, s2, s1, a);
2375                                         break;
2376                                 case TYPE_FLT:
2377                                         var_to_reg_flt(s2, src, REG_FTMP2);
2378                                         gen_nullptr_check(s1);
2379                                         x86_64_movss_reg_membase(cd, s2, s1, a);
2380                                         break;
2381                                 case TYPE_DBL:
2382                                         var_to_reg_flt(s2, src, REG_FTMP2);
2383                                         gen_nullptr_check(s1);
2384                                         x86_64_movsd_reg_membase(cd, s2, s1, a);
2385                                         break;
2386                                 default: panic ("internal error");
2387                                 }
2388                         break;
2389
2390                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2391                                       /* op1 = type, val.i = field offset             */
2392
2393                         a = ((fieldinfo *)(iptr->val.a))->offset;
2394                         var_to_reg_int(s1, src, REG_ITMP1);
2395                         switch (iptr->op1) {
2396                                 case TYPE_INT:
2397                                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2398                                         gen_nullptr_check(s1);
2399                                         x86_64_movl_membase_reg(cd, s1, a, d);
2400                                         store_reg_to_var_int(iptr->dst, d);
2401                                         break;
2402                                 case TYPE_LNG:
2403                                 case TYPE_ADR:
2404                                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2405                                         gen_nullptr_check(s1);
2406                                         x86_64_mov_membase_reg(cd, s1, a, d);
2407                                         store_reg_to_var_int(iptr->dst, d);
2408                                         break;
2409                                 case TYPE_FLT:
2410                                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2411                                         gen_nullptr_check(s1);
2412                                         x86_64_movss_membase_reg(cd, s1, a, d);
2413                                         store_reg_to_var_flt(iptr->dst, d);
2414                                         break;
2415                                 case TYPE_DBL:
2416                                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2417                                         gen_nullptr_check(s1);
2418                                         x86_64_movsd_membase_reg(cd, s1, a, d);
2419                                         store_reg_to_var_flt(iptr->dst, d);
2420                                         break;
2421                                 default: panic ("internal error");
2422                                 }
2423                         break;
2424
2425
2426                 /* branch operations **************************************************/
2427
2428                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2429
2430                         var_to_reg_int(s1, src, REG_ITMP1);
2431                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2432
2433                         x86_64_call_imm(cd, 0); /* passing exception pointer                  */
2434                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2435
2436                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
2437                         x86_64_jmp_reg(cd, REG_ITMP3);
2438                         ALIGNCODENOP;
2439                         break;
2440
2441                 case ICMD_GOTO:         /* ... ==> ...                                */
2442                                         /* op1 = target JavaVM pc                     */
2443
2444                         x86_64_jmp_imm(cd, 0);
2445                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2446                         ALIGNCODENOP;
2447                         break;
2448
2449                 case ICMD_JSR:          /* ... ==> ...                                */
2450                                         /* op1 = target JavaVM pc                     */
2451
2452                         x86_64_call_imm(cd, 0);
2453                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2454                         break;
2455
2456                 case ICMD_RET:          /* ... ==> ...                                */
2457                                         /* op1 = local variable                       */
2458
2459                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2460                         var_to_reg_int(s1, var, REG_ITMP1);
2461                         x86_64_jmp_reg(cd, s1);
2462                         break;
2463
2464                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2465                                         /* op1 = target JavaVM pc                     */
2466
2467                         if (src->flags & INMEMORY) {
2468                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2469
2470                         } else {
2471                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2472                         }
2473                         x86_64_jcc(cd, X86_64_CC_E, 0);
2474                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2475                         break;
2476
2477                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2478                                         /* op1 = target JavaVM pc                     */
2479
2480                         if (src->flags & INMEMORY) {
2481                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2482
2483                         } else {
2484                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2485                         }
2486                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2487                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2488                         break;
2489
2490                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2491                                         /* op1 = target JavaVM pc, val.i = constant   */
2492
2493                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2494                         break;
2495
2496                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2497                                         /* op1 = target JavaVM pc, val.i = constant   */
2498
2499                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2500                         break;
2501
2502                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2503                                         /* op1 = target JavaVM pc, val.i = constant   */
2504
2505                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2506                         break;
2507
2508                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2509                                         /* op1 = target JavaVM pc, val.i = constant   */
2510
2511                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2512                         break;
2513
2514                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2515                                         /* op1 = target JavaVM pc, val.i = constant   */
2516
2517                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2518                         break;
2519
2520                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2521                                         /* op1 = target JavaVM pc, val.i = constant   */
2522
2523                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2524                         break;
2525
2526                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2527                                         /* op1 = target JavaVM pc, val.l = constant   */
2528
2529                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2530                         break;
2531
2532                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2533                                         /* op1 = target JavaVM pc, val.l = constant   */
2534
2535                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2536                         break;
2537
2538                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2539                                         /* op1 = target JavaVM pc, val.l = constant   */
2540
2541                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2542                         break;
2543
2544                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2545                                         /* op1 = target JavaVM pc, val.l = constant   */
2546
2547                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2548                         break;
2549
2550                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2551                                         /* op1 = target JavaVM pc, val.l = constant   */
2552
2553                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2554                         break;
2555
2556                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2557                                         /* op1 = target JavaVM pc, val.l = constant   */
2558
2559                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2560                         break;
2561
2562                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2563                                         /* op1 = target JavaVM pc                     */
2564
2565                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2566                         break;
2567
2568                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2569                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2570
2571                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2572                         break;
2573
2574                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2575                                         /* op1 = target JavaVM pc                     */
2576
2577                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2578                         break;
2579
2580                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2581                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2582
2583                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2584                         break;
2585
2586                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2587                                         /* op1 = target JavaVM pc                     */
2588
2589                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2590                         break;
2591
2592                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2593                                     /* op1 = target JavaVM pc                     */
2594
2595                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2596                         break;
2597
2598                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2599                                         /* op1 = target JavaVM pc                     */
2600
2601                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2602                         break;
2603
2604                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2605                                 /* op1 = target JavaVM pc                     */
2606
2607                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2608                         break;
2609
2610                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2611                                         /* op1 = target JavaVM pc                     */
2612
2613                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2614                         break;
2615
2616                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2617                                         /* op1 = target JavaVM pc                     */
2618
2619                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2620                         break;
2621
2622                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2623                                         /* op1 = target JavaVM pc                     */
2624
2625                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2626                         break;
2627
2628                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2629                                     /* op1 = target JavaVM pc                     */
2630
2631                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2632                         break;
2633
2634                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2635
2636                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2637                         break;
2638
2639                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2640                                         /* val.i = constant                           */
2641
2642                         var_to_reg_int(s1, src, REG_ITMP1);
2643                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2644                         s3 = iptr->val.i;
2645                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2646                                 if (s1 == d) {
2647                                         M_INTMOVE(s1, REG_ITMP1);
2648                                         s1 = REG_ITMP1;
2649                                 }
2650                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2651                         }
2652                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2653                         x86_64_testl_reg_reg(cd, s1, s1);
2654                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2655                         store_reg_to_var_int(iptr->dst, d);
2656                         break;
2657
2658                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2659                                         /* val.i = constant                           */
2660
2661                         var_to_reg_int(s1, src, REG_ITMP1);
2662                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2663                         s3 = iptr->val.i;
2664                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2665                                 if (s1 == d) {
2666                                         M_INTMOVE(s1, REG_ITMP1);
2667                                         s1 = REG_ITMP1;
2668                                 }
2669                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2670                         }
2671                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2672                         x86_64_testl_reg_reg(cd, s1, s1);
2673                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2674                         store_reg_to_var_int(iptr->dst, d);
2675                         break;
2676
2677                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2678                                         /* val.i = constant                           */
2679
2680                         var_to_reg_int(s1, src, REG_ITMP1);
2681                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2682                         s3 = iptr->val.i;
2683                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2684                                 if (s1 == d) {
2685                                         M_INTMOVE(s1, REG_ITMP1);
2686                                         s1 = REG_ITMP1;
2687                                 }
2688                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2689                         }
2690                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2691                         x86_64_testl_reg_reg(cd, s1, s1);
2692                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2693                         store_reg_to_var_int(iptr->dst, d);
2694                         break;
2695
2696                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2697                                         /* val.i = constant                           */
2698
2699                         var_to_reg_int(s1, src, REG_ITMP1);
2700                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2701                         s3 = iptr->val.i;
2702                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2703                                 if (s1 == d) {
2704                                         M_INTMOVE(s1, REG_ITMP1);
2705                                         s1 = REG_ITMP1;
2706                                 }
2707                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2708                         }
2709                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2710                         x86_64_testl_reg_reg(cd, s1, s1);
2711                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2712                         store_reg_to_var_int(iptr->dst, d);
2713                         break;
2714
2715                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2716                                         /* val.i = constant                           */
2717
2718                         var_to_reg_int(s1, src, REG_ITMP1);
2719                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2720                         s3 = iptr->val.i;
2721                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2722                                 if (s1 == d) {
2723                                         M_INTMOVE(s1, REG_ITMP1);
2724                                         s1 = REG_ITMP1;
2725                                 }
2726                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2727                         }
2728                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2729                         x86_64_testl_reg_reg(cd, s1, s1);
2730                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2731                         store_reg_to_var_int(iptr->dst, d);
2732                         break;
2733
2734                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2735                                         /* val.i = constant                           */
2736
2737                         var_to_reg_int(s1, src, REG_ITMP1);
2738                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2739                         s3 = iptr->val.i;
2740                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2741                                 if (s1 == d) {
2742                                         M_INTMOVE(s1, REG_ITMP1);
2743                                         s1 = REG_ITMP1;
2744                                 }
2745                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2746                         }
2747                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2748                         x86_64_testl_reg_reg(cd, s1, s1);
2749                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2750                         store_reg_to_var_int(iptr->dst, d);
2751                         break;
2752
2753
2754                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2755                 case ICMD_LRETURN:
2756                 case ICMD_ARETURN:
2757
2758                         var_to_reg_int(s1, src, REG_RESULT);
2759                         M_INTMOVE(s1, REG_RESULT);
2760
2761 #if defined(USE_THREADS)
2762                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2763                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2764                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->maxmemuse * 8);
2765                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2766                                 x86_64_call_reg(cd, REG_ITMP1);
2767                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_RESULT);
2768                         }
2769 #endif
2770
2771                         goto nowperformreturn;
2772
2773                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2774                 case ICMD_DRETURN:
2775
2776                         var_to_reg_flt(s1, src, REG_FRESULT);
2777                         M_FLTMOVE(s1, REG_FRESULT);
2778
2779 #if defined(USE_THREADS)
2780                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2781                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2782                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->maxmemuse * 8);
2783                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2784                                 x86_64_call_reg(cd, REG_ITMP1);
2785                                 x86_64_movq_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_FRESULT);
2786                         }
2787 #endif
2788
2789                         goto nowperformreturn;
2790
2791                 case ICMD_RETURN:      /* ...  ==> ...                                */
2792
2793 #if defined(USE_THREADS)
2794                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2795                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
2796                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2797                                 x86_64_call_reg(cd, REG_ITMP1);
2798                         }
2799 #endif
2800
2801 nowperformreturn:
2802                         {
2803                         s4 i, p;
2804
2805                         p = parentargs_base;
2806
2807                         /* call trace function */
2808                         if (runverbose) {
2809                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2810
2811                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2812                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2813
2814                                 x86_64_mov_imm_reg(cd, (s8) m, rd->argintregs[0]);
2815                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2816                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2817                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2818
2819                                 x86_64_mov_imm_reg(cd, (s8) builtin_displaymethodstop, REG_ITMP1);
2820                                 x86_64_call_reg(cd, REG_ITMP1);
2821
2822                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2823                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2824
2825                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2826                         }
2827
2828                         /* restore saved registers                                        */
2829                         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
2830                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
2831                         }
2832                         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
2833                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
2834                         }
2835
2836                         /* deallocate stack                                               */
2837                         if (parentargs_base) {
2838                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2839                         }
2840
2841                         x86_64_ret(cd);
2842                         ALIGNCODENOP;
2843                         }
2844                         break;
2845
2846
2847                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2848                         {
2849                                 s4 i, l, *s4ptr;
2850                                 void **tptr;
2851
2852                                 tptr = (void **) iptr->target;
2853
2854                                 s4ptr = iptr->val.a;
2855                                 l = s4ptr[1];                          /* low     */
2856                                 i = s4ptr[2];                          /* high    */
2857
2858                                 var_to_reg_int(s1, src, REG_ITMP1);
2859                                 M_INTMOVE(s1, REG_ITMP1);
2860                                 if (l != 0) {
2861                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2862                                 }
2863                                 i = i - l + 1;
2864
2865                 /* range check */
2866                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2867                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2868
2869                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2870                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2871
2872                                 /* build jump table top down and use address of lowest entry */
2873
2874                 /* s4ptr += 3 + i; */
2875                                 tptr += i;
2876
2877                                 while (--i >= 0) {
2878                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
2879                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
2880                                         --tptr;
2881                                 }
2882
2883                                 /* length of dataseg after last dseg_addtarget is used by load */
2884
2885                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2886                                 dseg_adddata(cd, cd->mcodeptr);
2887                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2888                                 x86_64_jmp_reg(cd, REG_ITMP1);
2889                                 ALIGNCODENOP;
2890                         }
2891                         break;
2892
2893
2894                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2895                         {
2896                                 s4 i, l, val, *s4ptr;
2897                                 void **tptr;
2898
2899                                 tptr = (void **) iptr->target;
2900
2901                                 s4ptr = iptr->val.a;
2902                                 l = s4ptr[0];                          /* default  */
2903                                 i = s4ptr[1];                          /* count    */
2904
2905                                 MCODECHECK((i<<2)+8);
2906                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
2907                                 while (--i >= 0) {
2908                                         s4ptr += 2;
2909                                         ++tptr;
2910
2911                                         val = s4ptr[0];
2912                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
2913                                         x86_64_jcc(cd, X86_64_CC_E, 0);
2914                                         /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
2915                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2916                                 }
2917
2918                                 x86_64_jmp_imm(cd, 0);
2919                                 /* codegen_addreference(cd, BlockPtrOfPC(l), cd->mcodeptr); */
2920
2921                                 tptr = (void **) iptr->target;
2922                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
2923
2924                                 ALIGNCODENOP;
2925                         }
2926                         break;
2927
2928
2929                 case ICMD_BUILTIN3:     /* ..., arg1, arg2, arg3 ==> ...              */
2930                                         /* op1 = return type, val.a = function pointer*/
2931                         s3 = 3;
2932                         goto gen_method;
2933
2934                 case ICMD_BUILTIN2:     /* ..., arg1, arg2 ==> ...                    */
2935                                         /* op1 = return type, val.a = function pointer*/
2936                         s3 = 2;
2937                         goto gen_method;
2938
2939                 case ICMD_BUILTIN1:     /* ..., arg1 ==> ...                          */
2940                                         /* op1 = return type, val.a = function pointer*/
2941                         s3 = 1;
2942                         goto gen_method;
2943
2944                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2945                                         /* op1 = arg count, val.a = method pointer    */
2946
2947                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2948                                         /* op1 = arg count, val.a = method pointer    */
2949
2950                 case ICMD_INVOKEVIRTUAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2951                                         /* op1 = arg count, val.a = method pointer    */
2952
2953                 case ICMD_INVOKEINTERFACE:/*.., objectref, [arg1, [arg2 ...]] ==> ... */
2954                                         /* op1 = arg count, val.a = method pointer    */
2955
2956                         s3 = iptr->op1;
2957
2958 gen_method: {
2959                         methodinfo   *lm;
2960                         classinfo    *ci;
2961                         stackptr      tmpsrc;
2962                         s4 iarg, farg;
2963
2964                         MCODECHECK((s3 << 1) + 64);
2965
2966                         tmpsrc = src;
2967                         s2 = s3;
2968                         iarg = 0;
2969                         farg = 0;
2970
2971                         /* copy arguments to registers or stack location                  */
2972                         for (; --s3 >= 0; src = src->prev) {
2973                                 IS_INT_LNG_TYPE(src->type) ? iarg++ : farg++;
2974                         }
2975
2976                         src = tmpsrc;
2977                         s3 = s2;
2978
2979                         s2 = (iarg > INT_ARG_CNT) ? iarg - INT_ARG_CNT : 0 + (farg > FLT_ARG_CNT) ? farg - FLT_ARG_CNT : 0;
2980
2981                         for (; --s3 >= 0; src = src->prev) {
2982                                 IS_INT_LNG_TYPE(src->type) ? iarg-- : farg--;
2983                                 if (src->varkind == ARGVAR) {
2984                                         if (IS_INT_LNG_TYPE(src->type)) {
2985                                                 if (iarg >= INT_ARG_CNT) {
2986                                                         s2--;
2987                                                 }
2988                                         } else {
2989                                                 if (farg >= FLT_ARG_CNT) {
2990                                                         s2--;
2991                                                 }
2992                                         }
2993                                         continue;
2994                                 }
2995
2996                                 if (IS_INT_LNG_TYPE(src->type)) {
2997                                         if (iarg < INT_ARG_CNT) {
2998                                                 s1 = rd->argintregs[iarg];
2999                                                 var_to_reg_int(d, src, s1);
3000                                                 M_INTMOVE(d, s1);
3001
3002                                         } else {
3003                                                 var_to_reg_int(d, src, REG_ITMP1);
3004                                                 s2--;
3005                                                 x86_64_mov_reg_membase(cd, d, REG_SP, s2 * 8);
3006                                         }
3007
3008                                 } else {
3009                                         if (farg < FLT_ARG_CNT) {
3010                                                 s1 = rd->argfltregs[farg];
3011                                                 var_to_reg_flt(d, src, s1);
3012                                                 M_FLTMOVE(d, s1);
3013
3014                                         } else {
3015                                                 var_to_reg_flt(d, src, REG_FTMP1);
3016                                                 s2--;
3017                                                 x86_64_movq_reg_membase(cd, d, REG_SP, s2 * 8);
3018                                         }
3019                                 }
3020                         } /* end of for */
3021
3022                         lm = iptr->val.a;
3023                         switch (iptr->opc) {
3024                                 case ICMD_BUILTIN3:
3025                                 case ICMD_BUILTIN2:
3026                                 case ICMD_BUILTIN1:
3027
3028                                         a = (s8) lm;
3029                                         d = iptr->op1;
3030
3031                                         x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3032                                         x86_64_call_reg(cd, REG_ITMP1);
3033                                         break;
3034
3035                                 case ICMD_INVOKESTATIC:
3036
3037                                         a = (s8) lm->stubroutine;
3038                                         d = lm->returntype;
3039
3040                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3041                                         x86_64_call_reg(cd, REG_ITMP2);
3042                                         break;
3043
3044                                 case ICMD_INVOKESPECIAL:
3045
3046                                         a = (s8) lm->stubroutine;
3047                                         d = lm->returntype;
3048
3049                                         gen_nullptr_check(rd->argintregs[0]);    /* first argument contains pointer */
3050                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); /* access memory for hardware nullptr */
3051                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3052                                         x86_64_call_reg(cd, REG_ITMP2);
3053                                         break;
3054
3055                                 case ICMD_INVOKEVIRTUAL:
3056
3057                                         d = lm->returntype;
3058
3059                                         gen_nullptr_check(rd->argintregs[0]);
3060                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3061                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, OFFSET(vftbl_t, table[0]) + sizeof(methodptr) * lm->vftblindex, REG_ITMP1);
3062                                         x86_64_call_reg(cd, REG_ITMP1);
3063                                         break;
3064
3065                                 case ICMD_INVOKEINTERFACE:
3066
3067                                         ci = lm->class;
3068                                         d = lm->returntype;
3069
3070                                         gen_nullptr_check(rd->argintregs[0]);
3071                                         x86_64_mov_membase_reg(cd, rd->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3072                                         x86_64_mov_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, interfacetable[0]) - sizeof(methodptr) * ci->index, REG_ITMP2);
3073                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, sizeof(methodptr) * (lm - ci->methods), REG_ITMP1);
3074                                         x86_64_call_reg(cd, REG_ITMP1);
3075                                         break;
3076
3077                                 default:
3078                                         d = 0;
3079                                         error("Unkown ICMD-Command: %d", iptr->opc);
3080                                 }
3081
3082                         /* d contains return type */
3083
3084                         if (d != TYPE_VOID) {
3085                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3086                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3087                                         M_INTMOVE(REG_RESULT, s1);
3088                                         store_reg_to_var_int(iptr->dst, s1);
3089
3090                                 } else {
3091                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3092                                         M_FLTMOVE(REG_FRESULT, s1);
3093                                         store_reg_to_var_flt(iptr->dst, s1);
3094                                 }
3095                         }
3096                         }
3097                         break;
3098
3099
3100                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3101
3102                                       /* op1:   0 == array, 1 == class                */
3103                                       /* val.a: (classinfo*) superclass               */
3104
3105 /*          superclass is an interface:
3106  *
3107  *          return (sub != NULL) &&
3108  *                 (sub->vftbl->interfacetablelength > super->index) &&
3109  *                 (sub->vftbl->interfacetable[-super->index] != NULL);
3110  *
3111  *          superclass is a class:
3112  *
3113  *          return ((sub != NULL) && (0
3114  *                  <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3115  *                  super->vftbl->diffvall));
3116  */
3117
3118                         {
3119                         classinfo *super = (classinfo*) iptr->val.a;
3120
3121 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3122             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3123 #endif
3124
3125                         var_to_reg_int(s1, src, REG_ITMP1);
3126                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3127                         if (s1 == d) {
3128                                 M_INTMOVE(s1, REG_ITMP1);
3129                                 s1 = REG_ITMP1;
3130                         }
3131                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3132                         if (iptr->op1) {                               /* class/interface */
3133                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3134                                         x86_64_test_reg_reg(cd, s1, s1);
3135
3136                                         /* TODO: clean up this calculation */
3137                                         a = 3;    /* mov_membase_reg */
3138                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3139
3140                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3141                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3142
3143                                         a += 3;    /* sub */
3144                                         CALCIMMEDIATEBYTES(a, super->index);
3145
3146                                         a += 3;    /* test */
3147
3148                                         a += 6;    /* jcc */
3149                                         a += 3;    /* mov_membase_reg */
3150                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3151
3152                                         a += 3;    /* test */
3153                                         a += 4;    /* setcc */
3154
3155                                         x86_64_jcc(cd, X86_64_CC_E, a);
3156
3157                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3158                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3159                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3160                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3161
3162                                         /* TODO: clean up this calculation */
3163                                         a = 0;
3164                                         a += 3;    /* mov_membase_reg */
3165                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3166
3167                                         a += 3;    /* test */
3168                                         a += 4;    /* setcc */
3169
3170                                         x86_64_jcc(cd, X86_64_CC_LE, a);
3171                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP1);
3172                                         x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3173                                         x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3174
3175                                 } else {                                   /* class           */
3176                                         x86_64_test_reg_reg(cd, s1, s1);
3177
3178                                         /* TODO: clean up this calculation */
3179                                         a = 3;    /* mov_membase_reg */
3180                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3181
3182                                         a += 10;   /* mov_imm_reg */
3183
3184                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3185                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3186
3187                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3188                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3189
3190                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3191                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3192
3193                                         a += 3;    /* sub */
3194                                         a += 3;    /* xor */
3195                                         a += 3;    /* cmp */
3196                                         a += 4;    /* setcc */
3197
3198                                         x86_64_jcc(cd, X86_64_CC_E, a);
3199
3200                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3201                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3202 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3203                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3204 #endif
3205                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3206                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3207                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3208 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3209                     codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3210 #endif
3211                                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3212                                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3213                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3214                                         x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3215                                 }
3216                         }
3217                         else
3218                                 panic("internal error: no inlined array instanceof");
3219                         }
3220                         store_reg_to_var_int(iptr->dst, d);
3221                         break;
3222
3223                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3224
3225                                       /* op1:   0 == array, 1 == class                */
3226                                       /* val.a: (classinfo*) superclass               */
3227
3228 /*          superclass is an interface:
3229  *
3230  *          OK if ((sub == NULL) ||
3231  *                 (sub->vftbl->interfacetablelength > super->index) &&
3232  *                 (sub->vftbl->interfacetable[-super->index] != NULL));
3233  *
3234  *          superclass is a class:
3235  *
3236  *          OK if ((sub == NULL) || (0
3237  *                 <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3238  *                 super->vftbl->diffvall));
3239  */
3240
3241                         {
3242                         classinfo *super = (classinfo*) iptr->val.a;
3243
3244 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3245             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3246 #endif
3247                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3248                         var_to_reg_int(s1, src, d);
3249                         if (iptr->op1) {                               /* class/interface */
3250                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3251                                         x86_64_test_reg_reg(cd, s1, s1);
3252
3253                                         /* TODO: clean up this calculation */
3254                                         a = 3;    /* mov_membase_reg */
3255                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3256
3257                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3258                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3259
3260                                         a += 3;    /* sub */
3261                                         CALCIMMEDIATEBYTES(a, super->index);
3262
3263                                         a += 3;    /* test */
3264                                         a += 6;    /* jcc */
3265
3266                                         a += 3;    /* mov_membase_reg */
3267                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3268
3269                                         a += 3;    /* test */
3270                                         a += 6;    /* jcc */
3271
3272                                         x86_64_jcc(cd, X86_64_CC_E, a);
3273
3274                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3275                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3276                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3277                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3278                                         x86_64_jcc(cd, X86_64_CC_LE, 0);
3279                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3280                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP2);
3281                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3282                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3283                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3284
3285                                 } else {                                     /* class           */
3286                                         x86_64_test_reg_reg(cd, s1, s1);
3287
3288                                         /* TODO: clean up this calculation */
3289                                         a = 3;    /* mov_membase_reg */
3290                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3291                                         a += 10;   /* mov_imm_reg */
3292                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3293                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3294
3295                                         if (d != REG_ITMP3) {
3296                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3297                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3298                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3299                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3300                                                 a += 3;    /* sub */
3301
3302                                         } else {
3303                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3304                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3305                                                 a += 3;    /* sub */
3306                                                 a += 10;   /* mov_imm_reg */
3307                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3308                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3309                                         }
3310
3311                                         a += 3;    /* cmp */
3312                                         a += 6;    /* jcc */
3313
3314                                         x86_64_jcc(cd, X86_64_CC_E, a);
3315
3316                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3317                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3318 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3319                     codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3320 #endif
3321                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3322                                         if (d != REG_ITMP3) {
3323                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3324                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3325 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3326                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3327 #endif
3328                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3329
3330                                         } else {
3331                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP2);
3332                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3333                                                 x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3334                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3335 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3336                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3337 #endif
3338                                         }
3339                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3340                                         x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3341                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3342                                 }
3343
3344                         } else
3345                                 panic("internal error: no inlined array checkcast");
3346                         }
3347                         M_INTMOVE(s1, d);
3348                         store_reg_to_var_int(iptr->dst, d);
3349                         break;
3350
3351                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3352
3353                         if (src->flags & INMEMORY) {
3354                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3355
3356                         } else {
3357                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3358                         }
3359                         x86_64_jcc(cd, X86_64_CC_L, 0);
3360                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3361                         break;
3362
3363                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3364
3365                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3366                         x86_64_jcc(cd, X86_64_CC_E, 0);
3367                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3368                         break;
3369
3370                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3371                                          /* op1 = dimension, val.a = array descriptor */
3372
3373                         /* check for negative sizes and copy sizes to stack if necessary  */
3374
3375                         MCODECHECK((iptr->op1 << 1) + 64);
3376
3377                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3378                                 var_to_reg_int(s2, src, REG_ITMP1);
3379                                 x86_64_testl_reg_reg(cd, s2, s2);
3380                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3381                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3382
3383                                 /* copy sizes to stack (argument numbers >= INT_ARG_CNT)      */
3384
3385                                 if (src->varkind != ARGVAR) {
3386                                         x86_64_mov_reg_membase(cd, s2, REG_SP, (s1 + INT_ARG_CNT) * 8);
3387                                 }
3388                         }
3389
3390                         /* a0 = dimension count */
3391                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3392
3393                         /* a1 = arraydescriptor */
3394                         x86_64_mov_imm_reg(cd, (s8) iptr->val.a, rd->argintregs[1]);
3395
3396                         /* a2 = pointer to dimensions = stack pointer */
3397                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3398
3399                         x86_64_mov_imm_reg(cd, (s8) builtin_nmultianewarray, REG_ITMP1);
3400                         x86_64_call_reg(cd, REG_ITMP1);
3401
3402                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3403                         M_INTMOVE(REG_RESULT, s1);
3404                         store_reg_to_var_int(iptr->dst, s1);
3405                         break;
3406
3407                 default: error("Unknown pseudo command: %d", iptr->opc);
3408         } /* switch */
3409
3410         } /* for instruction */
3411
3412         /* copy values to interface registers */
3413
3414         src = bptr->outstack;
3415         len = bptr->outdepth;
3416         MCODECHECK(64 + len);
3417         while (src) {
3418                 len--;
3419                 if ((src->varkind != STACKVAR)) {
3420                         s2 = src->type;
3421                         if (IS_FLT_DBL_TYPE(s2)) {
3422                                 var_to_reg_flt(s1, src, REG_FTMP1);
3423                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3424                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3425
3426                                 } else {
3427                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3428                                 }
3429
3430                         } else {
3431                                 var_to_reg_int(s1, src, REG_ITMP1);
3432                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3433                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3434
3435                                 } else {
3436                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3437                                 }
3438                         }
3439                 }
3440                 src = src->prev;
3441         }
3442         } /* if (bptr -> flags >= BBREACHED) */
3443         } /* for basic block */
3444
3445         {
3446
3447         /* generate bound check stubs */
3448
3449         u1 *xcodeptr = NULL;
3450         branchref *bref;
3451
3452         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3453                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3454                                   bref->branchpos,
3455                                                   cd->mcodeptr - cd->mcodebase);
3456
3457                 MCODECHECK(50);
3458
3459                 /* move index register into REG_ITMP1 */
3460                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);              /* 3 bytes  */
3461
3462                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                  /* 10 bytes */
3463                 dseg_adddata(cd, cd->mcodeptr);
3464                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3465                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3466
3467                 if (xcodeptr != NULL) {
3468                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3469
3470                 } else {
3471                         xcodeptr = cd->mcodeptr;
3472
3473                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3474                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3475
3476                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3477                         x86_64_mov_imm_reg(cd, (s8) new_arrayindexoutofboundsexception, REG_ITMP3);
3478                         x86_64_call_reg(cd, REG_ITMP3);
3479
3480                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3481                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3482
3483                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3484                         x86_64_jmp_reg(cd, REG_ITMP3);
3485                 }
3486         }
3487
3488         /* generate negative array size check stubs */
3489
3490         xcodeptr = NULL;
3491
3492         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3493                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3494                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3495                                                           bref->branchpos,
3496                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3497                         continue;
3498                 }
3499
3500                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3501                                   bref->branchpos,
3502                                                   cd->mcodeptr - cd->mcodebase);
3503
3504                 MCODECHECK(50);
3505
3506                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                         /* 10 bytes */
3507                 dseg_adddata(cd, cd->mcodeptr);
3508                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3509                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);     /* 3 bytes  */
3510
3511                 if (xcodeptr != NULL) {
3512                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3513
3514                 } else {
3515                         xcodeptr = cd->mcodeptr;
3516
3517                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3518                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3519
3520                         x86_64_mov_imm_reg(cd, (s8) new_negativearraysizeexception, REG_ITMP3);
3521                         x86_64_call_reg(cd, REG_ITMP3);
3522
3523                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3524                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3525
3526                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3527                         x86_64_jmp_reg(cd, REG_ITMP3);
3528                 }
3529         }
3530
3531         /* generate cast check stubs */
3532
3533         xcodeptr = NULL;
3534
3535         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3536                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3537                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3538                                                           bref->branchpos,
3539                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3540                         continue;
3541                 }
3542
3543                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3544                                   bref->branchpos,
3545                                                   cd->mcodeptr - cd->mcodebase);
3546
3547                 MCODECHECK(50);
3548
3549                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3550                 dseg_adddata(cd, cd->mcodeptr);
3551                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);     /* 10 bytes */
3552                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3553
3554                 if (xcodeptr != NULL) {
3555                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3556
3557                 } else {
3558                         xcodeptr = cd->mcodeptr;
3559
3560                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3561                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3562
3563                         x86_64_mov_imm_reg(cd, (s8) new_classcastexception, REG_ITMP3);
3564                         x86_64_call_reg(cd, REG_ITMP3);
3565
3566                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3567                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3568
3569                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3570                         x86_64_jmp_reg(cd, REG_ITMP3);
3571                 }
3572         }
3573
3574         /* generate divide by zero check stubs */
3575
3576         xcodeptr = NULL;
3577
3578         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3579                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3580                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3581                                                           bref->branchpos,
3582                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3583                         continue;
3584                 }
3585
3586                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3587                                   bref->branchpos,
3588                                                   cd->mcodeptr - cd->mcodebase);
3589
3590                 MCODECHECK(50);
3591
3592                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3593                 dseg_adddata(cd, cd->mcodeptr);
3594                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);      /* 10 bytes */
3595                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3596
3597                 if (xcodeptr != NULL) {
3598                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3599
3600                 } else {
3601                         xcodeptr = cd->mcodeptr;
3602
3603                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3604                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3605
3606                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
3607                         x86_64_call_reg(cd, REG_ITMP3);
3608
3609                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3610                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3611
3612                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3613                         x86_64_jmp_reg(cd, REG_ITMP3);
3614                 }
3615         }
3616
3617         /* generate exception check stubs */
3618
3619         xcodeptr = NULL;
3620
3621         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3622                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3623                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3624                                                           bref->branchpos,
3625                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3626                         continue;
3627                 }
3628
3629                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3630                                   bref->branchpos,
3631                                                   cd->mcodeptr - cd->mcodebase);
3632
3633                 MCODECHECK(50);
3634
3635                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3636                 dseg_adddata(cd, cd->mcodeptr);
3637                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3638                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3639
3640                 if (xcodeptr != NULL) {
3641                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3642
3643                 } else {
3644                         xcodeptr = cd->mcodeptr;
3645
3646 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3647                         x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3648                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0);
3649                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3650                         x86_64_call_reg(cd, REG_ITMP1);
3651                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3652                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3653                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3654                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
3655                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
3656 #else
3657                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
3658                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
3659                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
3660 #endif
3661
3662                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3663                         x86_64_jmp_reg(cd, REG_ITMP3);
3664                 }
3665         }
3666
3667         /* generate null pointer check stubs */
3668
3669         xcodeptr = NULL;
3670
3671         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3672                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3673                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3674                                                           bref->branchpos,
3675                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3676                         continue;
3677                 }
3678
3679                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3680                                   bref->branchpos,
3681                                                   cd->mcodeptr - cd->mcodebase);
3682
3683                 MCODECHECK(50);
3684
3685                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3686                 dseg_adddata(cd, cd->mcodeptr);
3687                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3688                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3689
3690                 if (xcodeptr != NULL) {
3691                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3692
3693                 } else {
3694                         xcodeptr = cd->mcodeptr;
3695
3696                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3697                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3698
3699                         x86_64_mov_imm_reg(cd, (s8) new_nullpointerexception, REG_ITMP3);
3700                         x86_64_call_reg(cd, REG_ITMP3);
3701
3702                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3703                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3704
3705                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3706                         x86_64_jmp_reg(cd, REG_ITMP3);
3707                 }
3708         }
3709         }
3710
3711         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
3712 }
3713
3714
3715 /* function createcompilerstub *************************************************
3716
3717         creates a stub routine which calls the compiler
3718
3719 *******************************************************************************/
3720
3721 #define COMPSTUBSIZE 23
3722
3723 u1 *createcompilerstub(methodinfo *m)
3724 {
3725         u1 *s = CNEW(u1, COMPSTUBSIZE);     /* memory to hold the stub            */
3726         codegendata *cd = NEW(codegendata);
3727
3728         /* allocate code and data memory, with using inline stuff */
3729         //codegen_setup(m, cd, NULL);
3730         cd->mcodeptr = s;
3731
3732         /* code for the stub */
3733         x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1); /* pass method to compiler     */
3734         x86_64_mov_imm_reg(cd, (s8) asm_call_jit_compiler, REG_ITMP3);/* load address */
3735         x86_64_jmp_reg(cd, REG_ITMP3);      /* jump to compiler                   */
3736
3737 #if defined(STATISTICS)
3738         if (opt_stat)
3739                 count_cstub_len += COMPSTUBSIZE;
3740 #endif
3741
3742         /* free code and data memory */
3743         //codegen_close(m, cd);
3744
3745         FREE(cd, codegendata);
3746
3747         return s;
3748 }
3749
3750
3751 /* function removecompilerstub *************************************************
3752
3753      deletes a compilerstub from memory  (simply by freeing it)
3754
3755 *******************************************************************************/
3756
3757 void removecompilerstub(u1 *stub)
3758 {
3759         CFREE(stub, COMPSTUBSIZE);
3760 }
3761
3762
3763 /* function: createnativestub **************************************************
3764
3765         creates a stub routine which calls a native method
3766
3767 *******************************************************************************/
3768
3769 /* #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3770 /* static java_objectheader **(*callgetexceptionptrptr)() = builtin_get_exceptionptrptr; */
3771 /* #endif */
3772
3773 #define NATIVESTUBSIZE 420
3774
3775 u1 *createnativestub(functionptr f, methodinfo *m)
3776 {
3777         u1 *s = CNEW(u1, NATIVESTUBSIZE);   /* memory to hold the stub            */
3778         s4 stackframesize;                  /* size of stackframe if needed       */
3779         codegendata *cd = NEW(codegendata);
3780         registerdata *rd = NEW(registerdata);
3781         t_inlining_globals *id = NEW(t_inlining_globals);
3782
3783         /* setup registers before using it */
3784         inlining_setup(m, id);
3785         reg_setup(m, rd, id);
3786
3787         cd->mcodeptr = s;
3788
3789     descriptor2types(m);                /* set paramcount and paramtypes      */
3790
3791         /* if function is static, check for initialized */
3792
3793         if (m->flags & ACC_STATIC) {
3794                 /* if class isn't yet initialized, do it */
3795                 if (!m->class->initialized) {
3796                         /* call helper function which patches this code */
3797                         x86_64_mov_imm_reg(cd, (u8) m->class, REG_ITMP1);
3798                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP2);
3799                         x86_64_call_reg(cd, REG_ITMP2);
3800                 }
3801         }
3802
3803         if (runverbose) {
3804                 s4 p, l, s1;
3805
3806                 x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3807
3808                 x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 1 * 8);
3809                 x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 2 * 8);
3810                 x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 3 * 8);
3811                 x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 4 * 8);
3812                 x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 5 * 8);
3813                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 6 * 8);
3814
3815                 x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 7 * 8);
3816                 x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 8 * 8);
3817                 x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 9 * 8);
3818                 x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 10 * 8);
3819 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 11 * 8); */
3820 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 12 * 8); */
3821 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 13 * 8); */
3822 /*              x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 14 * 8); */
3823
3824                 /* show integer hex code for float arguments */
3825                 for (p = 0, l = 0; p < m->paramcount; p++) {
3826                         if (IS_FLT_DBL_TYPE(m->paramtypes[p])) {
3827                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
3828                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
3829                                 }
3830
3831                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
3832                                 l++;
3833                         }
3834                 }
3835
3836                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1);
3837                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
3838                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
3839                 x86_64_call_reg(cd, REG_ITMP1);
3840
3841                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[0]);
3842                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[1]);
3843                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[2]);
3844                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[3]);
3845                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[4]);
3846                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, rd->argintregs[5]);
3847
3848                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[0]);
3849                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[1]);
3850                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[2]);
3851                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[3]);
3852 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[4]); */
3853 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[5]); */
3854 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[6]); */
3855 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, rd->argfltregs[7]); */
3856
3857                 x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3858         }
3859
3860 #if 0
3861         x86_64_alu_imm_reg(cd, X86_64_SUB, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3862
3863         /* save callee saved float registers */
3864         x86_64_movq_reg_membase(cd, XMM15, REG_SP, 0 * 8);
3865         x86_64_movq_reg_membase(cd, XMM14, REG_SP, 1 * 8);
3866         x86_64_movq_reg_membase(cd, XMM13, REG_SP, 2 * 8);
3867         x86_64_movq_reg_membase(cd, XMM12, REG_SP, 3 * 8);
3868         x86_64_movq_reg_membase(cd, XMM11, REG_SP, 4 * 8);
3869         x86_64_movq_reg_membase(cd, XMM10, REG_SP, 5 * 8);
3870 #endif
3871
3872         /* save argument registers on stack -- if we have to */
3873         if ((m->flags & ACC_STATIC && m->paramcount > (INT_ARG_CNT - 2)) || m->paramcount > (INT_ARG_CNT - 1)) {
3874                 s4 i;
3875                 s4 paramshiftcnt = (m->flags & ACC_STATIC) ? 2 : 1;
3876                 s4 stackparamcnt = (m->paramcount > INT_ARG_CNT) ? m->paramcount - INT_ARG_CNT : 0;
3877
3878                 stackframesize = stackparamcnt + paramshiftcnt;
3879
3880                 /* keep stack 16-byte aligned */
3881                 if (!(stackframesize & 0x1))
3882                         stackframesize++;
3883
3884                 x86_64_alu_imm_reg(cd, X86_64_SUB, stackframesize * 8, REG_SP);
3885
3886                 /* copy stack arguments into new stack frame -- if any */
3887                 for (i = 0; i < stackparamcnt; i++) {
3888                         x86_64_mov_membase_reg(cd, REG_SP, (stackparamcnt + 1 + i) * 8, REG_ITMP1);
3889                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, (paramshiftcnt + i) * 8);
3890                 }
3891
3892                 if (m->flags & ACC_STATIC) {
3893                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 1 * 8);
3894                         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 0 * 8);
3895
3896                 } else {
3897                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 0 * 8);
3898                 }
3899
3900         } else {
3901                 /* keep stack 16-byte aligned */
3902                 x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3903                 stackframesize = 1;
3904         }
3905
3906         if (m->flags & ACC_STATIC) {
3907                 x86_64_mov_reg_reg(cd, rd->argintregs[3], rd->argintregs[5]);
3908                 x86_64_mov_reg_reg(cd, rd->argintregs[2], rd->argintregs[4]);
3909                 x86_64_mov_reg_reg(cd, rd->argintregs[1], rd->argintregs[3]);
3910                 x86_64_mov_reg_reg(cd, rd->argintregs[0], rd->argintregs[2]);
3911
3912                 /* put class into second argument register */
3913                 x86_64_mov_imm_reg(cd, (u8) m->class, rd->argintregs[1]);
3914
3915         } else {
3916                 x86_64_mov_reg_reg(cd, rd->argintregs[4], rd->argintregs[5]);
3917                 x86_64_mov_reg_reg(cd, rd->argintregs[3], rd->argintregs[4]);
3918                 x86_64_mov_reg_reg(cd, rd->argintregs[2], rd->argintregs[3]);
3919                 x86_64_mov_reg_reg(cd, rd->argintregs[1], rd->argintregs[2]);
3920                 x86_64_mov_reg_reg(cd, rd->argintregs[0], rd->argintregs[1]);
3921         }
3922
3923         /* put env into first argument register */
3924         x86_64_mov_imm_reg(cd, (u8) &env, rd->argintregs[0]);
3925
3926         x86_64_mov_imm_reg(cd, (u8) f, REG_ITMP1);
3927         x86_64_call_reg(cd, REG_ITMP1);
3928
3929         /* remove stackframe if there is one */
3930         if (stackframesize) {
3931                 x86_64_alu_imm_reg(cd, X86_64_ADD, stackframesize * 8, REG_SP);
3932         }
3933
3934         if (runverbose) {
3935                 x86_64_alu_imm_reg(cd, X86_64_SUB, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
3936
3937                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
3938                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
3939
3940                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
3941                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
3942                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
3943                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
3944
3945                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
3946                 x86_64_call_reg(cd, REG_ITMP1);
3947
3948                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
3949                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
3950
3951                 x86_64_alu_imm_reg(cd, X86_64_ADD, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
3952         }
3953
3954 #if 0
3955         /* restore callee saved registers */
3956         x86_64_movq_membase_reg(cd, REG_SP, 0 * 8, XMM15);
3957         x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, XMM14);
3958         x86_64_movq_membase_reg(cd, REG_SP, 2 * 8, XMM13);
3959         x86_64_movq_membase_reg(cd, REG_SP, 3 * 8, XMM12);
3960         x86_64_movq_membase_reg(cd, REG_SP, 4 * 8, XMM11);
3961         x86_64_movq_membase_reg(cd, REG_SP, 5 * 8, XMM10);
3962
3963         x86_64_alu_imm_reg(cd, X86_64_ADD, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3964 #endif
3965
3966 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3967         x86_64_push_reg(cd, REG_RESULT);
3968 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
3969         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
3970         x86_64_call_reg(cd, REG_ITMP3);
3971         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3972         x86_64_pop_reg(cd, REG_RESULT);
3973 #else
3974         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
3975         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
3976 #endif
3977         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3978         x86_64_jcc(cd, X86_64_CC_NE, 1);
3979
3980         x86_64_ret(cd);
3981
3982 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3983         x86_64_push_reg(cd, REG_ITMP3);
3984 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
3985         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
3986         x86_64_call_reg(cd, REG_ITMP3);
3987         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3988         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
3989 #else
3990         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3991         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
3992         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
3993         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
3994 #endif
3995
3996         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
3997         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
3998
3999         x86_64_mov_imm_reg(cd, (s8) asm_handle_nat_exception, REG_ITMP3);
4000         x86_64_jmp_reg(cd, REG_ITMP3);
4001
4002 #if 0
4003         {
4004                 static int stubprinted;
4005                 if (!stubprinted)
4006                         printf("stubsize: %d\n", ((long) cd->mcodeptr - (long) s));
4007                 stubprinted = 1;
4008         }
4009 #endif
4010
4011 #if defined(STATISTICS)
4012         if (opt_stat)
4013                 count_nstub_len += NATIVESTUBSIZE;
4014 #endif
4015
4016         FREE(cd, codegendata);
4017         FREE(rd, registerdata);
4018
4019         return s;
4020 }
4021
4022
4023 /* function: removenativestub **************************************************
4024
4025     removes a previously created native-stub from memory
4026
4027 *******************************************************************************/
4028
4029 void removenativestub(u1 *stub)
4030 {
4031         CFREE(stub, NATIVESTUBSIZE);
4032 }
4033
4034
4035 /*
4036  * These are local overrides for various environment variables in Emacs.
4037  * Please do not remove this and leave it at the end of the file, where
4038  * Emacs will automagically detect them.
4039  * ---------------------------------------------------------------------
4040  * Local variables:
4041  * mode: c
4042  * indent-tabs-mode: t
4043  * c-basic-offset: 4
4044  * tab-width: 4
4045  * End:
4046  */