src/vm/jit/x86_64/codegen.c

   1 /* jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003
   4    Institut f. Computersprachen, TU Wien
   5    R. Grafl, A. Krall, C. Kruegel, C. Oates, R. Obermaisser, M. Probst,
   6    S. Ring, E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich,
   7    J. Wenninger
   8
   9    This file is part of CACAO.
  10
  11    This program is free software; you can redistribute it and/or
  12    modify it under the terms of the GNU General Public License as
  13    published by the Free Software Foundation; either version 2, or (at
  14    your option) any later version.
  15
  16    This program is distributed in the hope that it will be useful, but
  17    WITHOUT ANY WARRANTY; without even the implied warranty of
  18    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19    General Public License for more details.
  20
  21    You should have received a copy of the GNU General Public License
  22    along with this program; if not, write to the Free Software
  23    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  24    02111-1307, USA.
  25
  26    Contact: cacao@complang.tuwien.ac.at
  27
  28    Authors: Andreas Krall
  29             Christian Thalinger
  30
  31    $Id: codegen.c 1367 2004-08-01 07:29:50Z stefan $
  32
  33 */
  34
  35 #define _GNU_SOURCE
  36
  37 #include "global.h"
  38 #include <stdio.h>
  39 #include <signal.h>
  40 #include <sys/ucontext.h>
  41 #include "builtin.h"
  42 #include "asmpart.h"
  43 #include "jni.h"
  44 #include "loader.h"
  45 #include "tables.h"
  46 #include "native.h"
  47 #include "jit/jit.h"
  48 #include "jit/reg.h"
  49 #include "jit/parse.h"
  50 #include "jit/x86_64/codegen.h"
  51 #include "jit/x86_64/emitfuncs.h"
  52 #include "jit/x86_64/types.h"
  53
  54 /* include independent code generation stuff */
  55 #include "jit/codegen.inc"
  56 #include "jit/reg.inc"
  57
  58
  59 /* register descripton - array ************************************************/
  60
  61 /* #define REG_RES   0         reserved register for OS or code generator     */
  62 /* #define REG_RET   1         return value register                          */
  63 /* #define REG_EXC   2         exception value register (only old jit)        */
  64 /* #define REG_SAV   3         (callee) saved register                        */
  65 /* #define REG_TMP   4         scratch temporary register (caller saved)      */
  66 /* #define REG_ARG   5         argument register (caller saved)               */
  67
  68 /* #define REG_END   -1        last entry in tables                           */
  69
  70 int nregdescint[] = {
  71     REG_RET, REG_ARG, REG_ARG, REG_TMP, REG_RES, REG_SAV, REG_ARG, REG_ARG,
  72     REG_ARG, REG_ARG, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV,
  73     REG_END
  74 };
  75
  76
  77 int nregdescfloat[] = {
  78         /*      REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP, */
  79         /*      REG_RES, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV, REG_SAV, */
  80     REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  81     REG_RES, REG_RES, REG_RES, REG_TMP, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  82     REG_END
  83 };
  84
  85
  86 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
  87 void thread_restartcriticalsection(ucontext_t *uc)
  88 {
  89         void *critical;
  90
  91         critical = thread_checkcritical((void *) uc->uc_mcontext.gregs[REG_RIP]);
  92
  93         if (critical)
  94                 uc->uc_mcontext.gregs[REG_RIP] = (u8) critical;
  95 }
  96 #endif
  97
  98
  99 /* NullPointerException signal handler for hardware null pointer check */
 100
 101 void catch_NullPointerException(int sig, siginfo_t *siginfo, void *_p)
 102 {
 103         sigset_t nsig;
 104         /*      int      instr; */
 105         /*      long     faultaddr; */
 106
 107         struct ucontext *_uc = (struct ucontext *) _p;
 108         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 109         struct sigaction act;
 110         java_objectheader *xptr;
 111
 112         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 113
 114
 115 /*      instr = *((int*)(sigctx->rip)); */
 116 /*      faultaddr = sigctx->sc_regs[(instr >> 16) & 0x1f]; */
 117
 118 /*      if (faultaddr == 0) { */
 119         act.sa_sigaction = (void *) catch_NullPointerException; /* reinstall handler */
 120         act.sa_flags = SA_SIGINFO;
 121         sigaction(sig, &act, NULL);
 122
 123         sigemptyset(&nsig);
 124         sigaddset(&nsig, sig);
 125         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 126
 127         xptr = new_exception(string_java_lang_NullPointerException);
 128
 129         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 130         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 131         sigctx->rip = (u8) asm_handle_exception;
 132
 133         return;
 134
 135 /*      } else { */
 136 /*              faultaddr += (long) ((instr << 16) >> 16); */
 137 /*              fprintf(stderr, "faulting address: 0x%08x\n", faultaddr); */
 138 /*              panic("Stack overflow"); */
 139 /*      } */
 140 }
 141
 142
 143 /* ArithmeticException signal handler for hardware divide by zero check */
 144
 145 void catch_ArithmeticException(int sig, siginfo_t *siginfo, void *_p)
 146 {
 147         sigset_t nsig;
 148
 149         struct ucontext *_uc = (struct ucontext *) _p;
 150         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 151         struct sigaction act;
 152         java_objectheader *xptr;
 153
 154         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 155
 156         act.sa_sigaction = (void *) catch_ArithmeticException; /* reinstall handler */
 157         act.sa_flags = SA_SIGINFO;
 158         sigaction(sig, &act, NULL);
 159
 160         sigemptyset(&nsig);
 161         sigaddset(&nsig, sig);
 162         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 163
 164         xptr = new_exception_message(string_java_lang_ArithmeticException,
 165                                                                  string_java_lang_ArithmeticException_message);
 166
 167         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 168         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 169         sigctx->rip = (u8) asm_handle_exception;
 170
 171         return;
 172 }
 173
 174
 175 void init_exceptions(void)
 176 {
 177         struct sigaction act;
 178
 179         /* install signal handlers we need to convert to exceptions */
 180         sigemptyset(&act.sa_mask);
 181
 182         if (!checknull) {
 183 #if defined(SIGSEGV)
 184                 act.sa_sigaction = (void *) catch_NullPointerException;
 185                 act.sa_flags = SA_SIGINFO;
 186                 sigaction(SIGSEGV, &act, NULL);
 187 #endif
 188
 189 #if defined(SIGBUS)
 190                 act.sa_sigaction = (void *) catch_NullPointerException;
 191                 act.sa_flags = SA_SIGINFO;
 192                 sigaction(SIGBUS, &act, NULL);
 193 #endif
 194         }
 195
 196         act.sa_sigaction = (void *) catch_ArithmeticException;
 197         act.sa_flags = SA_SIGINFO;
 198         sigaction(SIGFPE, &act, NULL);
 199 }
 200
 201
 202 /* function gen_mcode **********************************************************
 203
 204         generates machine code
 205
 206 *******************************************************************************/
 207
 208 void codegen(methodinfo *m)
 209 {
 210         s4 len, s1, s2, s3, d;
 211         s8 a;
 212         s4 parentargs_base;
 213         stackptr        src;
 214         varinfo        *var;
 215         basicblock     *bptr;
 216         instruction    *iptr;
 217         exceptiontable *ex;
 218         registerdata   *r;
 219         codegendata    *cd;
 220
 221         /* keep code size smaller */
 222         r = m->registerdata;
 223         cd = m->codegendata;
 224
 225         {
 226         s4 i, p, pa, t, l;
 227         s4 savedregs_num;
 228
 229         savedregs_num = 0;
 230
 231         /* space to save used callee saved registers */
 232
 233         savedregs_num += (r->savintregcnt - r->maxsavintreguse);
 234         savedregs_num += (r->savfltregcnt - r->maxsavfltreguse);
 235
 236         parentargs_base = r->maxmemuse + savedregs_num;
 237
 238 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 239
 240         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 241                 parentargs_base++;
 242
 243 #endif
 244
 245     /* keep stack 16-byte aligned for calls into libc */
 246
 247         if (!m->isleafmethod || runverbose) {
 248                 if ((parentargs_base % 2) == 0) {
 249                         parentargs_base++;
 250                 }
 251         }
 252
 253         /* create method header */
 254
 255         (void) dseg_addaddress(m, m);                           /* MethodPointer  */
 256         (void) dseg_adds4(m, parentargs_base * 8);              /* FrameSize      */
 257
 258 #if defined(USE_THREADS)
 259
 260         /* IsSync contains the offset relative to the stack pointer for the
 261            argument of monitor_exit used in the exception handler. Since the
 262            offset could be zero and give a wrong meaning of the flag it is
 263            offset by one.
 264         */
 265
 266         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 267                 (void) dseg_adds4(m, (r->maxmemuse + 1) * 8);       /* IsSync         */
 268         else
 269
 270 #endif
 271
 272                 (void) dseg_adds4(m, 0);                            /* IsSync         */
 273
 274         (void) dseg_adds4(m, m->isleafmethod);                  /* IsLeaf         */
 275         (void) dseg_adds4(m, r->savintregcnt - r->maxsavintreguse);/* IntSave     */
 276         (void) dseg_adds4(m, r->savfltregcnt - r->maxsavfltreguse);/* FltSave     */
 277         (void) dseg_adds4(m, m->exceptiontablelength);          /* ExTableSize    */
 278
 279         /* create exception table */
 280
 281         for (ex = m->exceptiontable; ex != NULL; ex = ex->down) {
 282                 dseg_addtarget(m, ex->start);
 283                 dseg_addtarget(m, ex->end);
 284                 dseg_addtarget(m, ex->handler);
 285                 (void) dseg_addaddress(m, ex->catchtype);
 286         }
 287
 288         /* initialize mcode variables */
 289
 290         cd->mcodeptr = (u1 *) cd->mcodebase;
 291         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 292         MCODECHECK(128 + m->paramcount);
 293
 294         /* create stack frame (if necessary) */
 295
 296         if (parentargs_base) {
 297                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 298         }
 299
 300         /* save return address and used callee saved registers */
 301
 302         p = parentargs_base;
 303         for (i = r->savintregcnt - 1; i >= r->maxsavintreguse; i--) {
 304                 p--; x86_64_mov_reg_membase(cd, r->savintregs[i], REG_SP, p * 8);
 305         }
 306         for (i = r->savfltregcnt - 1; i >= r->maxsavfltreguse; i--) {
 307                 p--; x86_64_movq_reg_membase(cd, r->savfltregs[i], REG_SP, p * 8);
 308         }
 309
 310         /* save monitorenter argument */
 311
 312 #if defined(USE_THREADS)
 313         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 314                 if (m->flags & ACC_STATIC) {
 315                         x86_64_mov_imm_reg(cd, (s8) m->class, REG_ITMP1);
 316                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, r->maxmemuse * 8);
 317
 318                 } else {
 319                         x86_64_mov_reg_membase(cd, r->argintregs[0], REG_SP, r->maxmemuse * 8);
 320                 }
 321         }
 322 #endif
 323
 324         /* copy argument registers to stack and call trace function with pointer
 325            to arguments on stack.
 326         */
 327         if (runverbose) {
 328                 x86_64_alu_imm_reg(cd, X86_64_SUB, (6 + 8 + 1 + 1) * 8, REG_SP);
 329
 330                 x86_64_mov_reg_membase(cd, r->argintregs[0], REG_SP, 1 * 8);
 331                 x86_64_mov_reg_membase(cd, r->argintregs[1], REG_SP, 2 * 8);
 332                 x86_64_mov_reg_membase(cd, r->argintregs[2], REG_SP, 3 * 8);
 333                 x86_64_mov_reg_membase(cd, r->argintregs[3], REG_SP, 4 * 8);
 334                 x86_64_mov_reg_membase(cd, r->argintregs[4], REG_SP, 5 * 8);
 335                 x86_64_mov_reg_membase(cd, r->argintregs[5], REG_SP, 6 * 8);
 336
 337                 x86_64_movq_reg_membase(cd, r->argfltregs[0], REG_SP, 7 * 8);
 338                 x86_64_movq_reg_membase(cd, r->argfltregs[1], REG_SP, 8 * 8);
 339                 x86_64_movq_reg_membase(cd, r->argfltregs[2], REG_SP, 9 * 8);
 340                 x86_64_movq_reg_membase(cd, r->argfltregs[3], REG_SP, 10 * 8);
 341 /*              x86_64_movq_reg_membase(cd, r->argfltregs[4], REG_SP, 11 * 8); */
 342 /*              x86_64_movq_reg_membase(cd, r->argfltregs[5], REG_SP, 12 * 8); */
 343 /*              x86_64_movq_reg_membase(cd, r->argfltregs[6], REG_SP, 13 * 8); */
 344 /*              x86_64_movq_reg_membase(cd, r->argfltregs[7], REG_SP, 14 * 8); */
 345
 346                 for (p = 0, l = 0; p < m->paramcount; p++) {
 347                         t = m->paramtypes[p];
 348
 349                         if (IS_FLT_DBL_TYPE(t)) {
 350                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
 351                                         x86_64_mov_reg_reg(cd, r->argintregs[s1], r->argintregs[s1 + 1]);
 352                                 }
 353
 354                                 x86_64_movd_freg_reg(cd, r->argfltregs[l], r->argintregs[p]);
 355                                 l++;
 356                         }
 357                 }
 358
 359                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP2);
 360                 x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 361                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
 362                 x86_64_call_reg(cd, REG_ITMP1);
 363
 364                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, r->argintregs[0]);
 365                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, r->argintregs[1]);
 366                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, r->argintregs[2]);
 367                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, r->argintregs[3]);
 368                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, r->argintregs[4]);
 369                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, r->argintregs[5]);
 370
 371                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, r->argfltregs[0]);
 372                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, r->argfltregs[1]);
 373                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, r->argfltregs[2]);
 374                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, r->argfltregs[3]);
 375 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, r->argfltregs[4]); */
 376 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, r->argfltregs[5]); */
 377 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, r->argfltregs[6]); */
 378 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, r->argfltregs[7]); */
 379
 380                 x86_64_alu_imm_reg(cd, X86_64_ADD, (6 + 8 + 1 + 1) * 8, REG_SP);
 381         }
 382
 383         /* take arguments out of register or stack frame */
 384
 385         for (p = 0, l = 0, s1 = 0, s2 = 0; p < m->paramcount; p++) {
 386                 t = m->paramtypes[p];
 387                 var = &(r->locals[l][t]);
 388                 l++;
 389                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 390                         l++;
 391                 if (var->type < 0) {
 392                         if (IS_INT_LNG_TYPE(t)) {
 393                                 s1++;
 394                         } else {
 395                                 s2++;
 396                         }
 397                         continue;
 398                 }
 399                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 400                         if (s1 < INT_ARG_CNT) {                  /* register arguments    */
 401                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 402                                         M_INTMOVE(r->argintregs[s1], var->regoff);
 403
 404                                 } else {                             /* reg arg -> spilled    */
 405                                     x86_64_mov_reg_membase(cd, r->argintregs[s1], REG_SP, var->regoff * 8);
 406                                 }
 407
 408                         } else {                                 /* stack arguments       */
 409                                 pa = s1 - INT_ARG_CNT;
 410                                 if (s2 >= FLT_ARG_CNT) {
 411                                         pa += s2 - FLT_ARG_CNT;
 412                                 }
 413                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 414                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);    /* + 8 for return address */
 415                                 } else {                             /* stack arg -> spilled  */
 416                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_ITMP1);    /* + 8 for return address */
 417                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, var->regoff * 8);
 418                                 }
 419                         }
 420                         s1++;
 421
 422                 } else {                                     /* floating args         */
 423                         if (s2 < FLT_ARG_CNT) {                /* register arguments    */
 424                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 425                                         M_FLTMOVE(r->argfltregs[s2], var->regoff);
 426
 427                                 } else {                                         /* reg arg -> spilled    */
 428                                         x86_64_movq_reg_membase(cd, r->argfltregs[s2], REG_SP, var->regoff * 8);
 429                                 }
 430
 431                         } else {                                 /* stack arguments       */
 432                                 pa = s2 - FLT_ARG_CNT;
 433                                 if (s1 >= INT_ARG_CNT) {
 434                                         pa += s1 - INT_ARG_CNT;
 435                                 }
 436                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 437                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);
 438
 439                                 } else {
 440                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_FTMP1);
 441                                         x86_64_movq_reg_membase(cd, REG_FTMP1, REG_SP, var->regoff * 8);
 442                                 }
 443                         }
 444                         s2++;
 445                 }
 446         }  /* end for */
 447
 448         /* call monitorenter function */
 449
 450 #if defined(USE_THREADS)
 451         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 452                 s8 func_enter = (m->flags & ACC_STATIC) ?
 453                         (s8) builtin_staticmonitorenter : (s8) builtin_monitorenter;
 454                 x86_64_mov_membase_reg(cd, REG_SP, r->maxmemuse * 8, r->argintregs[0]);
 455                 x86_64_mov_imm_reg(cd, func_enter, REG_ITMP1);
 456                 x86_64_call_reg(cd, REG_ITMP1);
 457         }
 458 #endif
 459         }
 460
 461         /* end of header generation */
 462
 463         /* walk through all basic blocks */
 464         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 465
 466                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 467
 468                 if (bptr->flags >= BBREACHED) {
 469
 470                         /* branch resolving */
 471
 472                         branchref *bref;
 473                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 474                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 475                                                                   bref->branchpos,
 476                                                                   bptr->mpc);
 477                         }
 478
 479                 /* copy interface registers to their destination */
 480
 481                 src = bptr->instack;
 482                 len = bptr->indepth;
 483                 MCODECHECK(64 + len);
 484                 while (src != NULL) {
 485                         len--;
 486                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 487                                 if (bptr->type == BBTYPE_SBR) {
 488                                         d = reg_of_var(m, src, REG_ITMP1);
 489                                         x86_64_pop_reg(cd, d);
 490                                         store_reg_to_var_int(src, d);
 491
 492                                 } else if (bptr->type == BBTYPE_EXH) {
 493                                         d = reg_of_var(m, src, REG_ITMP1);
 494                                         M_INTMOVE(REG_ITMP1, d);
 495                                         store_reg_to_var_int(src, d);
 496                                 }
 497
 498                         } else {
 499                                 d = reg_of_var(m, src, REG_ITMP1);
 500                                 if ((src->varkind != STACKVAR)) {
 501                                         s2 = src->type;
 502                                         if (IS_FLT_DBL_TYPE(s2)) {
 503                                                 s1 = r->interfaces[len][s2].regoff;
 504                                                 if (!(r->interfaces[len][s2].flags & INMEMORY)) {
 505                                                         M_FLTMOVE(s1, d);
 506
 507                                                 } else {
 508                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 509                                                 }
 510                                                 store_reg_to_var_flt(src, d);
 511
 512                                         } else {
 513                                                 s1 = r->interfaces[len][s2].regoff;
 514                                                 if (!(r->interfaces[len][s2].flags & INMEMORY)) {
 515                                                         M_INTMOVE(s1, d);
 516
 517                                                 } else {
 518                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 519                                                 }
 520                                                 store_reg_to_var_int(src, d);
 521                                         }
 522                                 }
 523                         }
 524                         src = src->prev;
 525                 }
 526
 527                 /* walk through all instructions */
 528
 529                 src = bptr->instack;
 530                 len = bptr->icount;
 531                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 532
 533                         MCODECHECK(64);   /* an instruction usually needs < 64 words      */
 534                         switch (iptr->opc) {
 535
 536                         case ICMD_NOP:    /* ...  ==> ...                                 */
 537                                 break;
 538
 539                         case ICMD_NULLCHECKPOP: /* ..., objectref  ==> ...                */
 540                                 if (src->flags & INMEMORY) {
 541                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 542
 543                                 } else {
 544                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 545                                 }
 546                                 x86_64_jcc(cd, X86_64_CC_E, 0);
 547                                 codegen_addxnullrefs(m, cd->mcodeptr);
 548                                 break;
 549
 550                 /* constant operations ************************************************/
 551
 552                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 553                                       /* op1 = 0, val.i = constant                    */
 554
 555                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
 556                         if (iptr->val.i == 0) {
 557                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 558                         } else {
 559                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 560                         }
 561                         store_reg_to_var_int(iptr->dst, d);
 562                         break;
 563
 564                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 565                                       /* op1 = 0, val.a = constant                    */
 566
 567                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
 568                         if (iptr->val.a == 0) {
 569                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 570                         } else {
 571                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 572                         }
 573                         store_reg_to_var_int(iptr->dst, d);
 574                         break;
 575
 576                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 577                                       /* op1 = 0, val.l = constant                    */
 578
 579                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
 580                         if (iptr->val.l == 0) {
 581                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 582                         } else {
 583                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 584                         }
 585                         store_reg_to_var_int(iptr->dst, d);
 586                         break;
 587
 588                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 589                                       /* op1 = 0, val.f = constant                    */
 590
 591                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
 592                         a = dseg_addfloat(m, iptr->val.f);
 593                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 594                         store_reg_to_var_flt(iptr->dst, d);
 595                         break;
 596
 597                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 598                                       /* op1 = 0, val.d = constant                    */
 599
 600                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
 601                         a = dseg_adddouble(m, iptr->val.d);
 602                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 603                         store_reg_to_var_flt(iptr->dst, d);
 604                         break;
 605
 606
 607                 /* load/store operations **********************************************/
 608
 609                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 610                                       /* op1 = local variable                         */
 611
 612                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
 613                         if ((iptr->dst->varkind == LOCALVAR) &&
 614                             (iptr->dst->varnum == iptr->op1)) {
 615                                 break;
 616                         }
 617                         var = &(r->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 618                         if (var->flags & INMEMORY) {
 619                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 620                                 store_reg_to_var_int(iptr->dst, d);
 621
 622                         } else {
 623                                 if (iptr->dst->flags & INMEMORY) {
 624                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 625
 626                                 } else {
 627                                         M_INTMOVE(var->regoff, d);
 628                                 }
 629                         }
 630                         break;
 631
 632                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 633                 case ICMD_ALOAD:      /* op1 = local variable                         */
 634
 635                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
 636                         if ((iptr->dst->varkind == LOCALVAR) &&
 637                             (iptr->dst->varnum == iptr->op1)) {
 638                                 break;
 639                         }
 640                         var = &(r->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 641                         if (var->flags & INMEMORY) {
 642                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 643                                 store_reg_to_var_int(iptr->dst, d);
 644
 645                         } else {
 646                                 if (iptr->dst->flags & INMEMORY) {
 647                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 648
 649                                 } else {
 650                                         M_INTMOVE(var->regoff, d);
 651                                 }
 652                         }
 653                         break;
 654
 655                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 656                 case ICMD_DLOAD:      /* op1 = local variable                         */
 657
 658                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
 659                         if ((iptr->dst->varkind == LOCALVAR) &&
 660                             (iptr->dst->varnum == iptr->op1)) {
 661                                 break;
 662                         }
 663                         var = &(r->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 664                         if (var->flags & INMEMORY) {
 665                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 666                                 store_reg_to_var_flt(iptr->dst, d);
 667
 668                         } else {
 669                                 if (iptr->dst->flags & INMEMORY) {
 670                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 671
 672                                 } else {
 673                                         M_FLTMOVE(var->regoff, d);
 674                                 }
 675                         }
 676                         break;
 677
 678                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 679                 case ICMD_LSTORE:     /* op1 = local variable                         */
 680                 case ICMD_ASTORE:
 681
 682                         if ((src->varkind == LOCALVAR) &&
 683                             (src->varnum == iptr->op1)) {
 684                                 break;
 685                         }
 686                         var = &(r->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 687                         if (var->flags & INMEMORY) {
 688                                 var_to_reg_int(s1, src, REG_ITMP1);
 689                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 690
 691                         } else {
 692                                 var_to_reg_int(s1, src, var->regoff);
 693                                 M_INTMOVE(s1, var->regoff);
 694                         }
 695                         break;
 696
 697                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 698                 case ICMD_DSTORE:     /* op1 = local variable                         */
 699
 700                         if ((src->varkind == LOCALVAR) &&
 701                             (src->varnum == iptr->op1)) {
 702                                 break;
 703                         }
 704                         var = &(r->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 705                         if (var->flags & INMEMORY) {
 706                                 var_to_reg_flt(s1, src, REG_FTMP1);
 707                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 708
 709                         } else {
 710                                 var_to_reg_flt(s1, src, var->regoff);
 711                                 M_FLTMOVE(s1, var->regoff);
 712                         }
 713                         break;
 714
 715
 716                 /* pop/dup/swap operations ********************************************/
 717
 718                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 719
 720                 case ICMD_POP:        /* ..., value  ==> ...                          */
 721                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 722                         break;
 723
 724                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 725                         M_COPY(src, iptr->dst);
 726                         break;
 727
 728                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 729
 730                         M_COPY(src,       iptr->dst);
 731                         M_COPY(src->prev, iptr->dst->prev);
 732                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 733                         break;
 734
 735                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 736
 737                         M_COPY(src,             iptr->dst);
 738                         M_COPY(src->prev,       iptr->dst->prev);
 739                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 740                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 741                         break;
 742
 743                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 744
 745                         M_COPY(src,       iptr->dst);
 746                         M_COPY(src->prev, iptr->dst->prev);
 747                         break;
 748
 749                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 750
 751                         M_COPY(src,             iptr->dst);
 752                         M_COPY(src->prev,       iptr->dst->prev);
 753                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 754                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 755                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 756                         break;
 757
 758                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 759
 760                         M_COPY(src,                   iptr->dst);
 761                         M_COPY(src->prev,             iptr->dst->prev);
 762                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 763                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 764                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 765                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 766                         break;
 767
 768                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 769
 770                         M_COPY(src,       iptr->dst->prev);
 771                         M_COPY(src->prev, iptr->dst);
 772                         break;
 773
 774
 775                 /* integer operations *************************************************/
 776
 777                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 778
 779                         d = reg_of_var(m, iptr->dst, REG_NULL);
 780                         if (iptr->dst->flags & INMEMORY) {
 781                                 if (src->flags & INMEMORY) {
 782                                         if (src->regoff == iptr->dst->regoff) {
 783                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 784
 785                                         } else {
 786                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 787                                                 x86_64_negl_reg(cd, REG_ITMP1);
 788                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 789                                         }
 790
 791                                 } else {
 792                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 793                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 794                                 }
 795
 796                         } else {
 797                                 if (src->flags & INMEMORY) {
 798                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 799                                         x86_64_negl_reg(cd, d);
 800
 801                                 } else {
 802                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 803                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 804                                 }
 805                         }
 806                         break;
 807
 808                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 809
 810                         d = reg_of_var(m, iptr->dst, REG_NULL);
 811                         if (iptr->dst->flags & INMEMORY) {
 812                                 if (src->flags & INMEMORY) {
 813                                         if (src->regoff == iptr->dst->regoff) {
 814                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 815
 816                                         } else {
 817                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 818                                                 x86_64_neg_reg(cd, REG_ITMP1);
 819                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 820                                         }
 821
 822                                 } else {
 823                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 824                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 825                                 }
 826
 827                         } else {
 828                                 if (src->flags & INMEMORY) {
 829                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 830                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 831
 832                                 } else {
 833                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 834                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 835                                 }
 836                         }
 837                         break;
 838
 839                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 840
 841                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
 842                         if (src->flags & INMEMORY) {
 843                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 844
 845                         } else {
 846                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 847                         }
 848                         store_reg_to_var_int(iptr->dst, d);
 849                         break;
 850
 851                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 852
 853                         var_to_reg_int(s1, src, REG_ITMP1);
 854                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
 855                         M_INTMOVE(s1, d);
 856                         store_reg_to_var_int(iptr->dst, d);
 857                         break;
 858
 859                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 860
 861                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
 862                         if (src->flags & INMEMORY) {
 863                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 864
 865                         } else {
 866                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 867                         }
 868                         store_reg_to_var_int(iptr->dst, d);
 869                         break;
 870
 871                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 872
 873                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
 874                         if (src->flags & INMEMORY) {
 875                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 876
 877                         } else {
 878                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 879                         }
 880                         store_reg_to_var_int(iptr->dst, d);
 881                         break;
 882
 883                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 884
 885                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
 886                         if (src->flags & INMEMORY) {
 887                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 888
 889                         } else {
 890                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 891                         }
 892                         store_reg_to_var_int(iptr->dst, d);
 893                         break;
 894
 895
 896                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 897
 898                         d = reg_of_var(m, iptr->dst, REG_NULL);
 899                         x86_64_emit_ialu(m, X86_64_ADD, src, iptr);
 900                         break;
 901
 902                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 903                                       /* val.i = constant                             */
 904
 905                         d = reg_of_var(m, iptr->dst, REG_NULL);
 906                         x86_64_emit_ialuconst(m, X86_64_ADD, src, iptr);
 907                         break;
 908
 909                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 910
 911                         d = reg_of_var(m, iptr->dst, REG_NULL);
 912                         x86_64_emit_lalu(m, X86_64_ADD, src, iptr);
 913                         break;
 914
 915                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 916                                       /* val.l = constant                             */
 917
 918                         d = reg_of_var(m, iptr->dst, REG_NULL);
 919                         x86_64_emit_laluconst(m, X86_64_ADD, src, iptr);
 920                         break;
 921
 922                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 923
 924                         d = reg_of_var(m, iptr->dst, REG_NULL);
 925                         if (iptr->dst->flags & INMEMORY) {
 926                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 927                                         if (src->prev->regoff == iptr->dst->regoff) {
 928                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 929                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 930
 931                                         } else {
 932                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 933                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 934                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 935                                         }
 936
 937                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 938                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 939                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 940                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 941
 942                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 943                                         if (src->prev->regoff == iptr->dst->regoff) {
 944                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 945
 946                                         } else {
 947                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 948                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 949                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 950                                         }
 951
 952                                 } else {
 953                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 954                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 955                                 }
 956
 957                         } else {
 958                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 959                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 960                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 961
 962                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 963                                         M_INTMOVE(src->prev->regoff, d);
 964                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 965
 966                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 967                                         /* workaround for reg alloc */
 968                                         if (src->regoff == iptr->dst->regoff) {
 969                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 970                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 971                                                 M_INTMOVE(REG_ITMP1, d);
 972
 973                                         } else {
 974                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 975                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 976                                         }
 977
 978                                 } else {
 979                                         /* workaround for reg alloc */
 980                                         if (src->regoff == iptr->dst->regoff) {
 981                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 982                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 983                                                 M_INTMOVE(REG_ITMP1, d);
 984
 985                                         } else {
 986                                                 M_INTMOVE(src->prev->regoff, d);
 987                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 988                                         }
 989                                 }
 990                         }
 991                         break;
 992
 993                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 994                                       /* val.i = constant                             */
 995
 996                         d = reg_of_var(m, iptr->dst, REG_NULL);
 997                         x86_64_emit_ialuconst(m, X86_64_SUB, src, iptr);
 998                         break;
 999
1000                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1001
1002                         d = reg_of_var(m, iptr->dst, REG_NULL);
1003                         if (iptr->dst->flags & INMEMORY) {
1004                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1005                                         if (src->prev->regoff == iptr->dst->regoff) {
1006                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1007                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1008
1009                                         } else {
1010                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1011                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1012                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1013                                         }
1014
1015                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1016                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1017                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1018                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1019
1020                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1021                                         if (src->prev->regoff == iptr->dst->regoff) {
1022                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1023
1024                                         } else {
1025                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1026                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1027                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1028                                         }
1029
1030                                 } else {
1031                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1032                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1033                                 }
1034
1035                         } else {
1036                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1037                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1038                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1039
1040                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1041                                         M_INTMOVE(src->prev->regoff, d);
1042                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1043
1044                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1045                                         /* workaround for reg alloc */
1046                                         if (src->regoff == iptr->dst->regoff) {
1047                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1048                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1049                                                 M_INTMOVE(REG_ITMP1, d);
1050
1051                                         } else {
1052                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1053                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1054                                         }
1055
1056                                 } else {
1057                                         /* workaround for reg alloc */
1058                                         if (src->regoff == iptr->dst->regoff) {
1059                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1060                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1061                                                 M_INTMOVE(REG_ITMP1, d);
1062
1063                                         } else {
1064                                                 M_INTMOVE(src->prev->regoff, d);
1065                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1066                                         }
1067                                 }
1068                         }
1069                         break;
1070
1071                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1072                                       /* val.l = constant                             */
1073
1074                         d = reg_of_var(m, iptr->dst, REG_NULL);
1075                         x86_64_emit_laluconst(m, X86_64_SUB, src, iptr);
1076                         break;
1077
1078                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1079
1080                         d = reg_of_var(m, iptr->dst, REG_NULL);
1081                         if (iptr->dst->flags & INMEMORY) {
1082                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1083                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1084                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1085                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1086
1087                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1088                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1089                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1090                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1091
1092                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1093                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1094                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1095                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1096
1097                                 } else {
1098                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1099                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1100                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1101                                 }
1102
1103                         } else {
1104                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1105                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1106                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1107
1108                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1109                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1110                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1111
1112                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1113                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1114                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1115
1116                                 } else {
1117                                         if (src->regoff == iptr->dst->regoff) {
1118                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1119
1120                                         } else {
1121                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1122                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1123                                         }
1124                                 }
1125                         }
1126                         break;
1127
1128                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1129                                       /* val.i = constant                             */
1130
1131                         d = reg_of_var(m, iptr->dst, REG_NULL);
1132                         if (iptr->dst->flags & INMEMORY) {
1133                                 if (src->flags & INMEMORY) {
1134                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1135                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1136
1137                                 } else {
1138                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1139                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1140                                 }
1141
1142                         } else {
1143                                 if (src->flags & INMEMORY) {
1144                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1145
1146                                 } else {
1147                                         if (iptr->val.i == 2) {
1148                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1149                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1150
1151                                         } else {
1152                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1153                                         }
1154                                 }
1155                         }
1156                         break;
1157
1158                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1159
1160                         d = reg_of_var(m, iptr->dst, REG_NULL);
1161                         if (iptr->dst->flags & INMEMORY) {
1162                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1163                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1164                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1165                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1166
1167                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1168                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1169                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1170                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1171
1172                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1173                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1174                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1175                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1176
1177                                 } else {
1178                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1179                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1180                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1181                                 }
1182
1183                         } else {
1184                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1185                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1186                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1187
1188                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1189                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1190                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1191
1192                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1193                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1194                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1195
1196                                 } else {
1197                                         if (src->regoff == iptr->dst->regoff) {
1198                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1199
1200                                         } else {
1201                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1202                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1203                                         }
1204                                 }
1205                         }
1206                         break;
1207
1208                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1209                                       /* val.l = constant                             */
1210
1211                         d = reg_of_var(m, iptr->dst, REG_NULL);
1212                         if (iptr->dst->flags & INMEMORY) {
1213                                 if (src->flags & INMEMORY) {
1214                                         if (x86_64_is_imm32(iptr->val.l)) {
1215                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1216
1217                                         } else {
1218                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1219                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1220                                         }
1221                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1222
1223                                 } else {
1224                                         if (x86_64_is_imm32(iptr->val.l)) {
1225                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1226
1227                                         } else {
1228                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1229                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1230                                         }
1231                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1232                                 }
1233
1234                         } else {
1235                                 if (src->flags & INMEMORY) {
1236                                         if (x86_64_is_imm32(iptr->val.l)) {
1237                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1238
1239                                         } else {
1240                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1241                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1242                                         }
1243
1244                                 } else {
1245                                         /* should match in many cases */
1246                                         if (iptr->val.l == 2) {
1247                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1248                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1249
1250                                         } else {
1251                                                 if (x86_64_is_imm32(iptr->val.l)) {
1252                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1253
1254                                                 } else {
1255                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1256                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1257                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1258                                                 }
1259                                         }
1260                                 }
1261                         }
1262                         break;
1263
1264                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1265
1266                         d = reg_of_var(m, iptr->dst, REG_NULL);
1267                 if (src->prev->flags & INMEMORY) {
1268                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1269
1270                         } else {
1271                                 M_INTMOVE(src->prev->regoff, RAX);
1272                         }
1273
1274                         if (src->flags & INMEMORY) {
1275                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1276
1277                         } else {
1278                                 M_INTMOVE(src->regoff, REG_ITMP3);
1279                         }
1280                         gen_div_check(src);
1281
1282                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1283                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1284                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1285                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1286
1287                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1288                         x86_64_cltd(cd);
1289                         x86_64_idivl_reg(cd, REG_ITMP3);
1290
1291                         if (iptr->dst->flags & INMEMORY) {
1292                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1293                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1294
1295                         } else {
1296                                 M_INTMOVE(RAX, iptr->dst->regoff);
1297
1298                                 if (iptr->dst->regoff != RDX) {
1299                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1300                                 }
1301                         }
1302                         break;
1303
1304                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1305
1306                         d = reg_of_var(m, iptr->dst, REG_NULL);
1307                         if (src->prev->flags & INMEMORY) {
1308                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1309
1310                         } else {
1311                                 M_INTMOVE(src->prev->regoff, RAX);
1312                         }
1313
1314                         if (src->flags & INMEMORY) {
1315                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1316
1317                         } else {
1318                                 M_INTMOVE(src->regoff, REG_ITMP3);
1319                         }
1320                         gen_div_check(src);
1321
1322                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1323                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1324                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1325                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1326                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1327
1328                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1329                         x86_64_cltd(cd);
1330                         x86_64_idivl_reg(cd, REG_ITMP3);
1331
1332                         if (iptr->dst->flags & INMEMORY) {
1333                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1334                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1335
1336                         } else {
1337                                 M_INTMOVE(RDX, iptr->dst->regoff);
1338
1339                                 if (iptr->dst->regoff != RDX) {
1340                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1341                                 }
1342                         }
1343                         break;
1344
1345                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1346                                       /* val.i = constant                             */
1347
1348                         var_to_reg_int(s1, src, REG_ITMP1);
1349                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1350                         M_INTMOVE(s1, REG_ITMP1);
1351                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1352                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1353                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1354                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1355                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1356                         store_reg_to_var_int(iptr->dst, d);
1357                         break;
1358
1359                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1360                                       /* val.i = constant                             */
1361
1362                         var_to_reg_int(s1, src, REG_ITMP1);
1363                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1364                         M_INTMOVE(s1, REG_ITMP1);
1365                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1366                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1367                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1368                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1369                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1370                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1371                         store_reg_to_var_int(iptr->dst, d);
1372                         break;
1373
1374
1375                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1376
1377                         d = reg_of_var(m, iptr->dst, REG_NULL);
1378                 if (src->prev->flags & INMEMORY) {
1379                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1380
1381                         } else {
1382                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1383                         }
1384
1385                         if (src->flags & INMEMORY) {
1386                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1387
1388                         } else {
1389                                 M_INTMOVE(src->regoff, REG_ITMP3);
1390                         }
1391                         gen_div_check(src);
1392
1393                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1394                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1395                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1396                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1397                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1398
1399                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1400                         x86_64_cqto(cd);
1401                         x86_64_idiv_reg(cd, REG_ITMP3);
1402
1403                         if (iptr->dst->flags & INMEMORY) {
1404                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1405                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1406
1407                         } else {
1408                                 M_INTMOVE(RAX, iptr->dst->regoff);
1409
1410                                 if (iptr->dst->regoff != RDX) {
1411                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1412                                 }
1413                         }
1414                         break;
1415
1416                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1417
1418                         d = reg_of_var(m, iptr->dst, REG_NULL);
1419                         if (src->prev->flags & INMEMORY) {
1420                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1421
1422                         } else {
1423                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1424                         }
1425
1426                         if (src->flags & INMEMORY) {
1427                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1428
1429                         } else {
1430                                 M_INTMOVE(src->regoff, REG_ITMP3);
1431                         }
1432                         gen_div_check(src);
1433
1434                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1435                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1436                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1437                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1438                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1439                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1440
1441                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1442                         x86_64_cqto(cd);
1443                         x86_64_idiv_reg(cd, REG_ITMP3);
1444
1445                         if (iptr->dst->flags & INMEMORY) {
1446                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1447                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1448
1449                         } else {
1450                                 M_INTMOVE(RDX, iptr->dst->regoff);
1451
1452                                 if (iptr->dst->regoff != RDX) {
1453                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1454                                 }
1455                         }
1456                         break;
1457
1458                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1459                                       /* val.i = constant                             */
1460
1461                         var_to_reg_int(s1, src, REG_ITMP1);
1462                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1463                         M_INTMOVE(s1, REG_ITMP1);
1464                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1465                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1466                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1467                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1468                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1469                         store_reg_to_var_int(iptr->dst, d);
1470                         break;
1471
1472                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1473                                       /* val.l = constant                             */
1474
1475                         var_to_reg_int(s1, src, REG_ITMP1);
1476                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1477                         M_INTMOVE(s1, REG_ITMP1);
1478                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1479                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1480                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1481                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1482                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1483                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1484                         store_reg_to_var_int(iptr->dst, d);
1485                         break;
1486
1487                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1488
1489                         d = reg_of_var(m, iptr->dst, REG_NULL);
1490                         x86_64_emit_ishift(m, X86_64_SHL, src, iptr);
1491                         break;
1492
1493                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1494                                       /* val.i = constant                             */
1495
1496                         d = reg_of_var(m, iptr->dst, REG_NULL);
1497                         x86_64_emit_ishiftconst(m, X86_64_SHL, src, iptr);
1498                         break;
1499
1500                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1501
1502                         d = reg_of_var(m, iptr->dst, REG_NULL);
1503                         x86_64_emit_ishift(m, X86_64_SAR, src, iptr);
1504                         break;
1505
1506                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1507                                       /* val.i = constant                             */
1508
1509                         d = reg_of_var(m, iptr->dst, REG_NULL);
1510                         x86_64_emit_ishiftconst(m, X86_64_SAR, src, iptr);
1511                         break;
1512
1513                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1514
1515                         d = reg_of_var(m, iptr->dst, REG_NULL);
1516                         x86_64_emit_ishift(m, X86_64_SHR, src, iptr);
1517                         break;
1518
1519                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1520                                       /* val.i = constant                             */
1521
1522                         d = reg_of_var(m, iptr->dst, REG_NULL);
1523                         x86_64_emit_ishiftconst(m, X86_64_SHR, src, iptr);
1524                         break;
1525
1526                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1527
1528                         d = reg_of_var(m, iptr->dst, REG_NULL);
1529                         x86_64_emit_lshift(m, X86_64_SHL, src, iptr);
1530                         break;
1531
1532         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1533                                           /* val.i = constant                             */
1534
1535                         d = reg_of_var(m, iptr->dst, REG_NULL);
1536                         x86_64_emit_lshiftconst(m, X86_64_SHL, src, iptr);
1537                         break;
1538
1539                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1540
1541                         d = reg_of_var(m, iptr->dst, REG_NULL);
1542                         x86_64_emit_lshift(m, X86_64_SAR, src, iptr);
1543                         break;
1544
1545                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1546                                       /* val.i = constant                             */
1547
1548                         d = reg_of_var(m, iptr->dst, REG_NULL);
1549                         x86_64_emit_lshiftconst(m, X86_64_SAR, src, iptr);
1550                         break;
1551
1552                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1553
1554                         d = reg_of_var(m, iptr->dst, REG_NULL);
1555                         x86_64_emit_lshift(m, X86_64_SHR, src, iptr);
1556                         break;
1557
1558                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1559                                       /* val.l = constant                             */
1560
1561                         d = reg_of_var(m, iptr->dst, REG_NULL);
1562                         x86_64_emit_lshiftconst(m, X86_64_SHR, src, iptr);
1563                         break;
1564
1565                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1566
1567                         d = reg_of_var(m, iptr->dst, REG_NULL);
1568                         x86_64_emit_ialu(m, X86_64_AND, src, iptr);
1569                         break;
1570
1571                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1572                                       /* val.i = constant                             */
1573
1574                         d = reg_of_var(m, iptr->dst, REG_NULL);
1575                         x86_64_emit_ialuconst(m, X86_64_AND, src, iptr);
1576                         break;
1577
1578                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1579
1580                         d = reg_of_var(m, iptr->dst, REG_NULL);
1581                         x86_64_emit_lalu(m, X86_64_AND, src, iptr);
1582                         break;
1583
1584                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1585                                       /* val.l = constant                             */
1586
1587                         d = reg_of_var(m, iptr->dst, REG_NULL);
1588                         x86_64_emit_laluconst(m, X86_64_AND, src, iptr);
1589                         break;
1590
1591                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1592
1593                         d = reg_of_var(m, iptr->dst, REG_NULL);
1594                         x86_64_emit_ialu(m, X86_64_OR, src, iptr);
1595                         break;
1596
1597                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1598                                       /* val.i = constant                             */
1599
1600                         d = reg_of_var(m, iptr->dst, REG_NULL);
1601                         x86_64_emit_ialuconst(m, X86_64_OR, src, iptr);
1602                         break;
1603
1604                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1605
1606                         d = reg_of_var(m, iptr->dst, REG_NULL);
1607                         x86_64_emit_lalu(m, X86_64_OR, src, iptr);
1608                         break;
1609
1610                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1611                                       /* val.l = constant                             */
1612
1613                         d = reg_of_var(m, iptr->dst, REG_NULL);
1614                         x86_64_emit_laluconst(m, X86_64_OR, src, iptr);
1615                         break;
1616
1617                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1618
1619                         d = reg_of_var(m, iptr->dst, REG_NULL);
1620                         x86_64_emit_ialu(m, X86_64_XOR, src, iptr);
1621                         break;
1622
1623                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1624                                       /* val.i = constant                             */
1625
1626                         d = reg_of_var(m, iptr->dst, REG_NULL);
1627                         x86_64_emit_ialuconst(m, X86_64_XOR, src, iptr);
1628                         break;
1629
1630                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1631
1632                         d = reg_of_var(m, iptr->dst, REG_NULL);
1633                         x86_64_emit_lalu(m, X86_64_XOR, src, iptr);
1634                         break;
1635
1636                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1637                                       /* val.l = constant                             */
1638
1639                         d = reg_of_var(m, iptr->dst, REG_NULL);
1640                         x86_64_emit_laluconst(m, X86_64_XOR, src, iptr);
1641                         break;
1642
1643
1644                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1645                                       /* op1 = variable, val.i = constant             */
1646
1647                         var = &(r->locals[iptr->op1][TYPE_INT]);
1648                         d = var->regoff;
1649                         if (var->flags & INMEMORY) {
1650                                 if (iptr->val.i == 1) {
1651                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1652
1653                                 } else if (iptr->val.i == -1) {
1654                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1655
1656                                 } else {
1657                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1658                                 }
1659
1660                         } else {
1661                                 if (iptr->val.i == 1) {
1662                                         x86_64_incl_reg(cd, d);
1663
1664                                 } else if (iptr->val.i == -1) {
1665                                         x86_64_decl_reg(cd, d);
1666
1667                                 } else {
1668                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1669                                 }
1670                         }
1671                         break;
1672
1673
1674                 /* floating operations ************************************************/
1675
1676                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1677
1678                         var_to_reg_flt(s1, src, REG_FTMP1);
1679                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1680                         a = dseg_adds4(m, 0x80000000);
1681                         M_FLTMOVE(s1, d);
1682                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1683                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1684                         store_reg_to_var_flt(iptr->dst, d);
1685                         break;
1686
1687                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1688
1689                         var_to_reg_flt(s1, src, REG_FTMP1);
1690                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1691                         a = dseg_adds8(m, 0x8000000000000000);
1692                         M_FLTMOVE(s1, d);
1693                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1694                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1695                         store_reg_to_var_flt(iptr->dst, d);
1696                         break;
1697
1698                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1699
1700                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1701                         var_to_reg_flt(s2, src, REG_FTMP2);
1702                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1703                         if (s1 == d) {
1704                                 x86_64_addss_reg_reg(cd, s2, d);
1705                         } else if (s2 == d) {
1706                                 x86_64_addss_reg_reg(cd, s1, d);
1707                         } else {
1708                                 M_FLTMOVE(s1, d);
1709                                 x86_64_addss_reg_reg(cd, s2, d);
1710                         }
1711                         store_reg_to_var_flt(iptr->dst, d);
1712                         break;
1713
1714                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1715
1716                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1717                         var_to_reg_flt(s2, src, REG_FTMP2);
1718                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1719                         if (s1 == d) {
1720                                 x86_64_addsd_reg_reg(cd, s2, d);
1721                         } else if (s2 == d) {
1722                                 x86_64_addsd_reg_reg(cd, s1, d);
1723                         } else {
1724                                 M_FLTMOVE(s1, d);
1725                                 x86_64_addsd_reg_reg(cd, s2, d);
1726                         }
1727                         store_reg_to_var_flt(iptr->dst, d);
1728                         break;
1729
1730                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1731
1732                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1733                         var_to_reg_flt(s2, src, REG_FTMP2);
1734                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1735                         if (s2 == d) {
1736                                 M_FLTMOVE(s2, REG_FTMP2);
1737                                 s2 = REG_FTMP2;
1738                         }
1739                         M_FLTMOVE(s1, d);
1740                         x86_64_subss_reg_reg(cd, s2, d);
1741                         store_reg_to_var_flt(iptr->dst, d);
1742                         break;
1743
1744                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1745
1746                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1747                         var_to_reg_flt(s2, src, REG_FTMP2);
1748                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1749                         if (s2 == d) {
1750                                 M_FLTMOVE(s2, REG_FTMP2);
1751                                 s2 = REG_FTMP2;
1752                         }
1753                         M_FLTMOVE(s1, d);
1754                         x86_64_subsd_reg_reg(cd, s2, d);
1755                         store_reg_to_var_flt(iptr->dst, d);
1756                         break;
1757
1758                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1759
1760                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1761                         var_to_reg_flt(s2, src, REG_FTMP2);
1762                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1763                         if (s1 == d) {
1764                                 x86_64_mulss_reg_reg(cd, s2, d);
1765                         } else if (s2 == d) {
1766                                 x86_64_mulss_reg_reg(cd, s1, d);
1767                         } else {
1768                                 M_FLTMOVE(s1, d);
1769                                 x86_64_mulss_reg_reg(cd, s2, d);
1770                         }
1771                         store_reg_to_var_flt(iptr->dst, d);
1772                         break;
1773
1774                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1775
1776                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1777                         var_to_reg_flt(s2, src, REG_FTMP2);
1778                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1779                         if (s1 == d) {
1780                                 x86_64_mulsd_reg_reg(cd, s2, d);
1781                         } else if (s2 == d) {
1782                                 x86_64_mulsd_reg_reg(cd, s1, d);
1783                         } else {
1784                                 M_FLTMOVE(s1, d);
1785                                 x86_64_mulsd_reg_reg(cd, s2, d);
1786                         }
1787                         store_reg_to_var_flt(iptr->dst, d);
1788                         break;
1789
1790                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1791
1792                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1793                         var_to_reg_flt(s2, src, REG_FTMP2);
1794                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1795                         if (s2 == d) {
1796                                 M_FLTMOVE(s2, REG_FTMP2);
1797                                 s2 = REG_FTMP2;
1798                         }
1799                         M_FLTMOVE(s1, d);
1800                         x86_64_divss_reg_reg(cd, s2, d);
1801                         store_reg_to_var_flt(iptr->dst, d);
1802                         break;
1803
1804                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1805
1806                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1807                         var_to_reg_flt(s2, src, REG_FTMP2);
1808                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1809                         if (s2 == d) {
1810                                 M_FLTMOVE(s2, REG_FTMP2);
1811                                 s2 = REG_FTMP2;
1812                         }
1813                         M_FLTMOVE(s1, d);
1814                         x86_64_divsd_reg_reg(cd, s2, d);
1815                         store_reg_to_var_flt(iptr->dst, d);
1816                         break;
1817
1818                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1819
1820                         var_to_reg_int(s1, src, REG_ITMP1);
1821                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
1822                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1823                         store_reg_to_var_flt(iptr->dst, d);
1824                         break;
1825
1826                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1827
1828                         var_to_reg_int(s1, src, REG_ITMP1);
1829                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
1830                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1831                         store_reg_to_var_flt(iptr->dst, d);
1832                         break;
1833
1834                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1835
1836                         var_to_reg_int(s1, src, REG_ITMP1);
1837                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
1838                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1839                         store_reg_to_var_flt(iptr->dst, d);
1840                         break;
1841
1842                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1843
1844                         var_to_reg_int(s1, src, REG_ITMP1);
1845                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
1846                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1847                         store_reg_to_var_flt(iptr->dst, d);
1848                         break;
1849
1850                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1851
1852                         var_to_reg_flt(s1, src, REG_FTMP1);
1853                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
1854                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1855                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1856                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1857                         x86_64_jcc(cd, X86_64_CC_NE, a);
1858                         M_FLTMOVE(s1, REG_FTMP1);
1859                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1860                         x86_64_call_reg(cd, REG_ITMP2);
1861                         M_INTMOVE(REG_RESULT, d);
1862                         store_reg_to_var_int(iptr->dst, d);
1863                         break;
1864
1865                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1866
1867                         var_to_reg_flt(s1, src, REG_FTMP1);
1868                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
1869                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1870                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1871                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1872                         x86_64_jcc(cd, X86_64_CC_NE, a);
1873                         M_FLTMOVE(s1, REG_FTMP1);
1874                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1875                         x86_64_call_reg(cd, REG_ITMP2);
1876                         M_INTMOVE(REG_RESULT, d);
1877                         store_reg_to_var_int(iptr->dst, d);
1878                         break;
1879
1880                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1881
1882                         var_to_reg_flt(s1, src, REG_FTMP1);
1883                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
1884                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1885                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1886                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1887                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1888                         x86_64_jcc(cd, X86_64_CC_NE, a);
1889                         M_FLTMOVE(s1, REG_FTMP1);
1890                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1891                         x86_64_call_reg(cd, REG_ITMP2);
1892                         M_INTMOVE(REG_RESULT, d);
1893                         store_reg_to_var_int(iptr->dst, d);
1894                         break;
1895
1896                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1897
1898                         var_to_reg_flt(s1, src, REG_FTMP1);
1899                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
1900                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1901                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1902                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1903                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1904                         x86_64_jcc(cd, X86_64_CC_NE, a);
1905                         M_FLTMOVE(s1, REG_FTMP1);
1906                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1907                         x86_64_call_reg(cd, REG_ITMP2);
1908                         M_INTMOVE(REG_RESULT, d);
1909                         store_reg_to_var_int(iptr->dst, d);
1910                         break;
1911
1912                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1913
1914                         var_to_reg_flt(s1, src, REG_FTMP1);
1915                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1916                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1917                         store_reg_to_var_flt(iptr->dst, d);
1918                         break;
1919
1920                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1921
1922                         var_to_reg_flt(s1, src, REG_FTMP1);
1923                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
1924                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1925                         store_reg_to_var_flt(iptr->dst, d);
1926                         break;
1927
1928                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1929                                           /* == => 0, < => 1, > => -1 */
1930
1931                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1932                         var_to_reg_flt(s2, src, REG_FTMP2);
1933                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1934                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1935                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1936                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1937                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1938                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1939                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1940                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1941                         store_reg_to_var_int(iptr->dst, d);
1942                         break;
1943
1944                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1945                                           /* == => 0, < => 1, > => -1 */
1946
1947                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1948                         var_to_reg_flt(s2, src, REG_FTMP2);
1949                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1950                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1951                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1952                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1953                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1954                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1955                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1956                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1957                         store_reg_to_var_int(iptr->dst, d);
1958                         break;
1959
1960                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1961                                           /* == => 0, < => 1, > => -1 */
1962
1963                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1964                         var_to_reg_flt(s2, src, REG_FTMP2);
1965                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1966                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1967                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1968                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1969                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1970                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1971                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1972                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1973                         store_reg_to_var_int(iptr->dst, d);
1974                         break;
1975
1976                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1977                                           /* == => 0, < => 1, > => -1 */
1978
1979                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1980                         var_to_reg_flt(s2, src, REG_FTMP2);
1981                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1982                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1983                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1984                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1985                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1986                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1987                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1988                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1989                         store_reg_to_var_int(iptr->dst, d);
1990                         break;
1991
1992
1993                 /* memory operations **************************************************/
1994
1995                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1996
1997                         var_to_reg_int(s1, src, REG_ITMP1);
1998                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
1999                         gen_nullptr_check(s1);
2000                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
2001                         store_reg_to_var_int(iptr->dst, d);
2002                         break;
2003
2004                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2005
2006                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2007                         var_to_reg_int(s2, src, REG_ITMP2);
2008                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2009                         if (iptr->op1 == 0) {
2010                                 gen_nullptr_check(s1);
2011                                 gen_bound_check;
2012                         }
2013                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2014                         store_reg_to_var_int(iptr->dst, d);
2015                         break;
2016
2017                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2018
2019                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2020                         var_to_reg_int(s2, src, REG_ITMP2);
2021                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2022                         if (iptr->op1 == 0) {
2023                                 gen_nullptr_check(s1);
2024                                 gen_bound_check;
2025                         }
2026                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2027                         store_reg_to_var_int(iptr->dst, d);
2028                         break;
2029
2030                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2031
2032                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2033                         var_to_reg_int(s2, src, REG_ITMP2);
2034                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2035                         if (iptr->op1 == 0) {
2036                                 gen_nullptr_check(s1);
2037                                 gen_bound_check;
2038                         }
2039                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2040                         store_reg_to_var_int(iptr->dst, d);
2041                         break;
2042
2043                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2044
2045                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2046                         var_to_reg_int(s2, src, REG_ITMP2);
2047                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
2048                         if (iptr->op1 == 0) {
2049                                 gen_nullptr_check(s1);
2050                                 gen_bound_check;
2051                         }
2052                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2053                         store_reg_to_var_flt(iptr->dst, d);
2054                         break;
2055
2056                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2057
2058                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2059                         var_to_reg_int(s2, src, REG_ITMP2);
2060                         d = reg_of_var(m, iptr->dst, REG_FTMP3);
2061                         if (iptr->op1 == 0) {
2062                                 gen_nullptr_check(s1);
2063                                 gen_bound_check;
2064                         }
2065                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2066                         store_reg_to_var_flt(iptr->dst, d);
2067                         break;
2068
2069                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2070
2071                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2072                         var_to_reg_int(s2, src, REG_ITMP2);
2073                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2074                         if (iptr->op1 == 0) {
2075                                 gen_nullptr_check(s1);
2076                                 gen_bound_check;
2077                         }
2078                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2079                         store_reg_to_var_int(iptr->dst, d);
2080                         break;
2081
2082                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2083
2084                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2085                         var_to_reg_int(s2, src, REG_ITMP2);
2086                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2087                         if (iptr->op1 == 0) {
2088                                 gen_nullptr_check(s1);
2089                                 gen_bound_check;
2090                         }
2091                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2092                         store_reg_to_var_int(iptr->dst, d);
2093                         break;
2094
2095                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2096
2097                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2098                         var_to_reg_int(s2, src, REG_ITMP2);
2099                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2100                         if (iptr->op1 == 0) {
2101                                 gen_nullptr_check(s1);
2102                                 gen_bound_check;
2103                         }
2104                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2105                         store_reg_to_var_int(iptr->dst, d);
2106                         break;
2107
2108
2109                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2110
2111                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2112                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2113                         if (iptr->op1 == 0) {
2114                                 gen_nullptr_check(s1);
2115                                 gen_bound_check;
2116                         }
2117                         var_to_reg_int(s3, src, REG_ITMP3);
2118                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2119                         break;
2120
2121                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2122
2123                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2124                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2125                         if (iptr->op1 == 0) {
2126                                 gen_nullptr_check(s1);
2127                                 gen_bound_check;
2128                         }
2129                         var_to_reg_int(s3, src, REG_ITMP3);
2130                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2131                         break;
2132
2133                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2134
2135                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2136                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2137                         if (iptr->op1 == 0) {
2138                                 gen_nullptr_check(s1);
2139                                 gen_bound_check;
2140                         }
2141                         var_to_reg_int(s3, src, REG_ITMP3);
2142                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2143                         break;
2144
2145                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2146
2147                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2148                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2149                         if (iptr->op1 == 0) {
2150                                 gen_nullptr_check(s1);
2151                                 gen_bound_check;
2152                         }
2153                         var_to_reg_flt(s3, src, REG_FTMP3);
2154                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2155                         break;
2156
2157                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2158
2159                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2160                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2161                         if (iptr->op1 == 0) {
2162                                 gen_nullptr_check(s1);
2163                                 gen_bound_check;
2164                         }
2165                         var_to_reg_flt(s3, src, REG_FTMP3);
2166                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2167                         break;
2168
2169                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2170
2171                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2172                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2173                         if (iptr->op1 == 0) {
2174                                 gen_nullptr_check(s1);
2175                                 gen_bound_check;
2176                         }
2177                         var_to_reg_int(s3, src, REG_ITMP3);
2178                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2179                         break;
2180
2181                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2182
2183                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2184                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2185                         if (iptr->op1 == 0) {
2186                                 gen_nullptr_check(s1);
2187                                 gen_bound_check;
2188                         }
2189                         var_to_reg_int(s3, src, REG_ITMP3);
2190                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2191                         break;
2192
2193                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2194
2195                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2196                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2197                         if (iptr->op1 == 0) {
2198                                 gen_nullptr_check(s1);
2199                                 gen_bound_check;
2200                         }
2201                         var_to_reg_int(s3, src, REG_ITMP3);
2202                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2203                         break;
2204
2205
2206                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2207                                       /* op1 = type, val.a = field address            */
2208
2209                         /* if class isn't yet initialized, do it */
2210                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2211                                 /* call helper function which patches this code */
2212                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2213                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2214                                 x86_64_call_reg(cd, REG_ITMP2);
2215                         }
2216
2217                         a = dseg_addaddress(m, &(((fieldinfo *) iptr->val.a)->value));
2218 /*                      x86_64_mov_imm_reg(cd, 0, REG_ITMP2); */
2219 /*                      dseg_adddata(m, cd->mcodeptr); */
2220 /*                      x86_64_mov_membase_reg(cd, REG_ITMP2, a, REG_ITMP2); */
2221                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2222                         switch (iptr->op1) {
2223                         case TYPE_INT:
2224                                 var_to_reg_int(s2, src, REG_ITMP1);
2225                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2226                                 break;
2227                         case TYPE_LNG:
2228                         case TYPE_ADR:
2229                                 var_to_reg_int(s2, src, REG_ITMP1);
2230                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2231                                 break;
2232                         case TYPE_FLT:
2233                                 var_to_reg_flt(s2, src, REG_FTMP1);
2234                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2235                                 break;
2236                         case TYPE_DBL:
2237                                 var_to_reg_flt(s2, src, REG_FTMP1);
2238                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2239                                 break;
2240                         default: panic("internal error");
2241                         }
2242                         break;
2243
2244                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2245                                       /* op1 = type, val.a = field address            */
2246
2247                         /* if class isn't yet initialized, do it */
2248                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2249                                 /* call helper function which patches this code */
2250                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2251                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2252                                 x86_64_call_reg(cd, REG_ITMP2);
2253                         }
2254
2255                         a = dseg_addaddress(m, &(((fieldinfo *) iptr->val.a)->value));
2256 /*                      x86_64_mov_imm_reg(cd, 0, REG_ITMP2); */
2257 /*                      dseg_adddata(m, cd->mcodeptr); */
2258 /*                      x86_64_mov_membase_reg(cd, REG_ITMP2, a, REG_ITMP2); */
2259                         x86_64_mov_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 7) - (s8) cd->mcodebase) + a, REG_ITMP2);
2260                         switch (iptr->op1) {
2261                         case TYPE_INT:
2262                                 d = reg_of_var(m, iptr->dst, REG_ITMP1);
2263                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2264                                 store_reg_to_var_int(iptr->dst, d);
2265                                 break;
2266                         case TYPE_LNG:
2267                         case TYPE_ADR:
2268                                 d = reg_of_var(m, iptr->dst, REG_ITMP1);
2269                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2270                                 store_reg_to_var_int(iptr->dst, d);
2271                                 break;
2272                         case TYPE_FLT:
2273                                 d = reg_of_var(m, iptr->dst, REG_ITMP1);
2274                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2275                                 store_reg_to_var_flt(iptr->dst, d);
2276                                 break;
2277                         case TYPE_DBL:
2278                                 d = reg_of_var(m, iptr->dst, REG_ITMP1);
2279                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2280                                 store_reg_to_var_flt(iptr->dst, d);
2281                                 break;
2282                         default: panic("internal error");
2283                         }
2284                         break;
2285
2286                 case ICMD_PUTFIELD:   /* ..., value  ==> ...                          */
2287                                       /* op1 = type, val.i = field offset             */
2288
2289                         /* if class isn't yet initialized, do it */
2290                         if (!((fieldinfo *) iptr->val.a)->class->initialized) {
2291                                 /* call helper function which patches this code */
2292                                 x86_64_mov_imm_reg(cd, (s8) ((fieldinfo *) iptr->val.a)->class, REG_ITMP1);
2293                                 x86_64_mov_imm_reg(cd, (s8) asm_check_clinit, REG_ITMP2);
2294                                 x86_64_call_reg(cd, REG_ITMP2);
2295                         }
2296
2297                         a = ((fieldinfo *)(iptr->val.a))->offset;
2298                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2299                         switch (iptr->op1) {
2300                                 case TYPE_INT:
2301                                         var_to_reg_int(s2, src, REG_ITMP2);
2302                                         gen_nullptr_check(s1);
2303                                         x86_64_movl_reg_membase(cd, s2, s1, a);
2304                                         break;
2305                                 case TYPE_LNG:
2306                                 case TYPE_ADR:
2307                                         var_to_reg_int(s2, src, REG_ITMP2);
2308                                         gen_nullptr_check(s1);
2309                                         x86_64_mov_reg_membase(cd, s2, s1, a);
2310                                         break;
2311                                 case TYPE_FLT:
2312                                         var_to_reg_flt(s2, src, REG_FTMP2);
2313                                         gen_nullptr_check(s1);
2314                                         x86_64_movss_reg_membase(cd, s2, s1, a);
2315                                         break;
2316                                 case TYPE_DBL:
2317                                         var_to_reg_flt(s2, src, REG_FTMP2);
2318                                         gen_nullptr_check(s1);
2319                                         x86_64_movsd_reg_membase(cd, s2, s1, a);
2320                                         break;
2321                                 default: panic ("internal error");
2322                                 }
2323                         break;
2324
2325                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2326                                       /* op1 = type, val.i = field offset             */
2327
2328                         a = ((fieldinfo *)(iptr->val.a))->offset;
2329                         var_to_reg_int(s1, src, REG_ITMP1);
2330                         switch (iptr->op1) {
2331                                 case TYPE_INT:
2332                                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
2333                                         gen_nullptr_check(s1);
2334                                         x86_64_movl_membase_reg(cd, s1, a, d);
2335                                         store_reg_to_var_int(iptr->dst, d);
2336                                         break;
2337                                 case TYPE_LNG:
2338                                 case TYPE_ADR:
2339                                         d = reg_of_var(m, iptr->dst, REG_ITMP1);
2340                                         gen_nullptr_check(s1);
2341                                         x86_64_mov_membase_reg(cd, s1, a, d);
2342                                         store_reg_to_var_int(iptr->dst, d);
2343                                         break;
2344                                 case TYPE_FLT:
2345                                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
2346                                         gen_nullptr_check(s1);
2347                                         x86_64_movss_membase_reg(cd, s1, a, d);
2348                                         store_reg_to_var_flt(iptr->dst, d);
2349                                         break;
2350                                 case TYPE_DBL:
2351                                         d = reg_of_var(m, iptr->dst, REG_FTMP1);
2352                                         gen_nullptr_check(s1);
2353                                         x86_64_movsd_membase_reg(cd, s1, a, d);
2354                                         store_reg_to_var_flt(iptr->dst, d);
2355                                         break;
2356                                 default: panic ("internal error");
2357                                 }
2358                         break;
2359
2360
2361                 /* branch operations **************************************************/
2362
2363                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2364
2365                         var_to_reg_int(s1, src, REG_ITMP1);
2366                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2367
2368                         x86_64_call_imm(cd, 0); /* passing exception pointer                  */
2369                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2370
2371                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
2372                         x86_64_jmp_reg(cd, REG_ITMP3);
2373                         ALIGNCODENOP;
2374                         break;
2375
2376                 case ICMD_GOTO:         /* ... ==> ...                                */
2377                                         /* op1 = target JavaVM pc                     */
2378
2379                         x86_64_jmp_imm(cd, 0);
2380                         codegen_addreference(m, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2381                         ALIGNCODENOP;
2382                         break;
2383
2384                 case ICMD_JSR:          /* ... ==> ...                                */
2385                                         /* op1 = target JavaVM pc                     */
2386
2387                         x86_64_call_imm(cd, 0);
2388                         codegen_addreference(m, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2389                         break;
2390
2391                 case ICMD_RET:          /* ... ==> ...                                */
2392                                         /* op1 = local variable                       */
2393
2394                         var = &(r->locals[iptr->op1][TYPE_ADR]);
2395                         var_to_reg_int(s1, var, REG_ITMP1);
2396                         x86_64_jmp_reg(cd, s1);
2397                         break;
2398
2399                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2400                                         /* op1 = target JavaVM pc                     */
2401
2402                         if (src->flags & INMEMORY) {
2403                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2404
2405                         } else {
2406                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2407                         }
2408                         x86_64_jcc(cd, X86_64_CC_E, 0);
2409                         codegen_addreference(m, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2410                         break;
2411
2412                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2413                                         /* op1 = target JavaVM pc                     */
2414
2415                         if (src->flags & INMEMORY) {
2416                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2417
2418                         } else {
2419                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2420                         }
2421                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2422                         codegen_addreference(m, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2423                         break;
2424
2425                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2426                                         /* op1 = target JavaVM pc, val.i = constant   */
2427
2428                         x86_64_emit_ifcc(m, X86_64_CC_E, src, iptr);
2429                         break;
2430
2431                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2432                                         /* op1 = target JavaVM pc, val.i = constant   */
2433
2434                         x86_64_emit_ifcc(m, X86_64_CC_L, src, iptr);
2435                         break;
2436
2437                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2438                                         /* op1 = target JavaVM pc, val.i = constant   */
2439
2440                         x86_64_emit_ifcc(m, X86_64_CC_LE, src, iptr);
2441                         break;
2442
2443                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2444                                         /* op1 = target JavaVM pc, val.i = constant   */
2445
2446                         x86_64_emit_ifcc(m, X86_64_CC_NE, src, iptr);
2447                         break;
2448
2449                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2450                                         /* op1 = target JavaVM pc, val.i = constant   */
2451
2452                         x86_64_emit_ifcc(m, X86_64_CC_G, src, iptr);
2453                         break;
2454
2455                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2456                                         /* op1 = target JavaVM pc, val.i = constant   */
2457
2458                         x86_64_emit_ifcc(m, X86_64_CC_GE, src, iptr);
2459                         break;
2460
2461                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2462                                         /* op1 = target JavaVM pc, val.l = constant   */
2463
2464                         x86_64_emit_if_lcc(m, X86_64_CC_E, src, iptr);
2465                         break;
2466
2467                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2468                                         /* op1 = target JavaVM pc, val.l = constant   */
2469
2470                         x86_64_emit_if_lcc(m, X86_64_CC_L, src, iptr);
2471                         break;
2472
2473                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2474                                         /* op1 = target JavaVM pc, val.l = constant   */
2475
2476                         x86_64_emit_if_lcc(m, X86_64_CC_LE, src, iptr);
2477                         break;
2478
2479                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2480                                         /* op1 = target JavaVM pc, val.l = constant   */
2481
2482                         x86_64_emit_if_lcc(m, X86_64_CC_NE, src, iptr);
2483                         break;
2484
2485                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2486                                         /* op1 = target JavaVM pc, val.l = constant   */
2487
2488                         x86_64_emit_if_lcc(m, X86_64_CC_G, src, iptr);
2489                         break;
2490
2491                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2492                                         /* op1 = target JavaVM pc, val.l = constant   */
2493
2494                         x86_64_emit_if_lcc(m, X86_64_CC_GE, src, iptr);
2495                         break;
2496
2497                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2498                                         /* op1 = target JavaVM pc                     */
2499
2500                         x86_64_emit_if_icmpcc(m, X86_64_CC_E, src, iptr);
2501                         break;
2502
2503                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2504                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2505
2506                         x86_64_emit_if_lcmpcc(m, X86_64_CC_E, src, iptr);
2507                         break;
2508
2509                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2510                                         /* op1 = target JavaVM pc                     */
2511
2512                         x86_64_emit_if_icmpcc(m, X86_64_CC_NE, src, iptr);
2513                         break;
2514
2515                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2516                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2517
2518                         x86_64_emit_if_lcmpcc(m, X86_64_CC_NE, src, iptr);
2519                         break;
2520
2521                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2522                                         /* op1 = target JavaVM pc                     */
2523
2524                         x86_64_emit_if_icmpcc(m, X86_64_CC_L, src, iptr);
2525                         break;
2526
2527                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2528                                     /* op1 = target JavaVM pc                     */
2529
2530                         x86_64_emit_if_lcmpcc(m, X86_64_CC_L, src, iptr);
2531                         break;
2532
2533                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2534                                         /* op1 = target JavaVM pc                     */
2535
2536                         x86_64_emit_if_icmpcc(m, X86_64_CC_G, src, iptr);
2537                         break;
2538
2539                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2540                                 /* op1 = target JavaVM pc                     */
2541
2542                         x86_64_emit_if_lcmpcc(m, X86_64_CC_G, src, iptr);
2543                         break;
2544
2545                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2546                                         /* op1 = target JavaVM pc                     */
2547
2548                         x86_64_emit_if_icmpcc(m, X86_64_CC_LE, src, iptr);
2549                         break;
2550
2551                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2552                                         /* op1 = target JavaVM pc                     */
2553
2554                         x86_64_emit_if_lcmpcc(m, X86_64_CC_LE, src, iptr);
2555                         break;
2556
2557                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2558                                         /* op1 = target JavaVM pc                     */
2559
2560                         x86_64_emit_if_icmpcc(m, X86_64_CC_GE, src, iptr);
2561                         break;
2562
2563                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2564                                     /* op1 = target JavaVM pc                     */
2565
2566                         x86_64_emit_if_lcmpcc(m, X86_64_CC_GE, src, iptr);
2567                         break;
2568
2569                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2570
2571                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2572                         break;
2573
2574                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2575                                         /* val.i = constant                           */
2576
2577                         var_to_reg_int(s1, src, REG_ITMP1);
2578                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2579                         s3 = iptr->val.i;
2580                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2581                                 if (s1 == d) {
2582                                         M_INTMOVE(s1, REG_ITMP1);
2583                                         s1 = REG_ITMP1;
2584                                 }
2585                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2586                         }
2587                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2588                         x86_64_testl_reg_reg(cd, s1, s1);
2589                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2590                         store_reg_to_var_int(iptr->dst, d);
2591                         break;
2592
2593                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2594                                         /* val.i = constant                           */
2595
2596                         var_to_reg_int(s1, src, REG_ITMP1);
2597                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2598                         s3 = iptr->val.i;
2599                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2600                                 if (s1 == d) {
2601                                         M_INTMOVE(s1, REG_ITMP1);
2602                                         s1 = REG_ITMP1;
2603                                 }
2604                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2605                         }
2606                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2607                         x86_64_testl_reg_reg(cd, s1, s1);
2608                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2609                         store_reg_to_var_int(iptr->dst, d);
2610                         break;
2611
2612                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2613                                         /* val.i = constant                           */
2614
2615                         var_to_reg_int(s1, src, REG_ITMP1);
2616                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2617                         s3 = iptr->val.i;
2618                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2619                                 if (s1 == d) {
2620                                         M_INTMOVE(s1, REG_ITMP1);
2621                                         s1 = REG_ITMP1;
2622                                 }
2623                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2624                         }
2625                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2626                         x86_64_testl_reg_reg(cd, s1, s1);
2627                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2628                         store_reg_to_var_int(iptr->dst, d);
2629                         break;
2630
2631                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2632                                         /* val.i = constant                           */
2633
2634                         var_to_reg_int(s1, src, REG_ITMP1);
2635                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2636                         s3 = iptr->val.i;
2637                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2638                                 if (s1 == d) {
2639                                         M_INTMOVE(s1, REG_ITMP1);
2640                                         s1 = REG_ITMP1;
2641                                 }
2642                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2643                         }
2644                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2645                         x86_64_testl_reg_reg(cd, s1, s1);
2646                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2647                         store_reg_to_var_int(iptr->dst, d);
2648                         break;
2649
2650                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2651                                         /* val.i = constant                           */
2652
2653                         var_to_reg_int(s1, src, REG_ITMP1);
2654                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2655                         s3 = iptr->val.i;
2656                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2657                                 if (s1 == d) {
2658                                         M_INTMOVE(s1, REG_ITMP1);
2659                                         s1 = REG_ITMP1;
2660                                 }
2661                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2662                         }
2663                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2664                         x86_64_testl_reg_reg(cd, s1, s1);
2665                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2666                         store_reg_to_var_int(iptr->dst, d);
2667                         break;
2668
2669                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2670                                         /* val.i = constant                           */
2671
2672                         var_to_reg_int(s1, src, REG_ITMP1);
2673                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
2674                         s3 = iptr->val.i;
2675                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2676                                 if (s1 == d) {
2677                                         M_INTMOVE(s1, REG_ITMP1);
2678                                         s1 = REG_ITMP1;
2679                                 }
2680                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2681                         }
2682                         x86_64_movl_imm_reg(cd, s3, REG_ITMP2);
2683                         x86_64_testl_reg_reg(cd, s1, s1);
2684                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2685                         store_reg_to_var_int(iptr->dst, d);
2686                         break;
2687
2688
2689                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2690                 case ICMD_LRETURN:
2691                 case ICMD_ARETURN:
2692
2693                         var_to_reg_int(s1, src, REG_RESULT);
2694                         M_INTMOVE(s1, REG_RESULT);
2695
2696 #if defined(USE_THREADS)
2697                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2698                                 x86_64_mov_membase_reg(cd, REG_SP, r->maxmemuse * 8, r->argintregs[0]);
2699                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, r->maxmemuse * 8);
2700                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2701                                 x86_64_call_reg(cd, REG_ITMP1);
2702                                 x86_64_mov_membase_reg(cd, REG_SP, r->maxmemuse * 8, REG_RESULT);
2703                         }
2704 #endif
2705
2706                         goto nowperformreturn;
2707
2708                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2709                 case ICMD_DRETURN:
2710
2711                         var_to_reg_flt(s1, src, REG_FRESULT);
2712                         M_FLTMOVE(s1, REG_FRESULT);
2713
2714 #if defined(USE_THREADS)
2715                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2716                                 x86_64_mov_membase_reg(cd, REG_SP, r->maxmemuse * 8, r->argintregs[0]);
2717                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, r->maxmemuse * 8);
2718                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2719                                 x86_64_call_reg(cd, REG_ITMP1);
2720                                 x86_64_movq_membase_reg(cd, REG_SP, r->maxmemuse * 8, REG_FRESULT);
2721                         }
2722 #endif
2723
2724                         goto nowperformreturn;
2725
2726                 case ICMD_RETURN:      /* ...  ==> ...                                */
2727
2728 #if defined(USE_THREADS)
2729                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2730                                 x86_64_mov_membase_reg(cd, REG_SP, r->maxmemuse * 8, r->argintregs[0]);
2731                                 x86_64_mov_imm_reg(cd, (u8) builtin_monitorexit, REG_ITMP1);
2732                                 x86_64_call_reg(cd, REG_ITMP1);
2733                         }
2734 #endif
2735
2736 nowperformreturn:
2737                         {
2738                         s4 i, p;
2739
2740                         p = parentargs_base;
2741
2742                         /* call trace function */
2743                         if (runverbose) {
2744                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2745
2746                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2747                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2748
2749                                 x86_64_mov_imm_reg(cd, (s8) m, r->argintregs[0]);
2750                                 x86_64_mov_reg_reg(cd, REG_RESULT, r->argintregs[1]);
2751                                 M_FLTMOVE(REG_FRESULT, r->argfltregs[0]);
2752                                 M_FLTMOVE(REG_FRESULT, r->argfltregs[1]);
2753
2754                                 x86_64_mov_imm_reg(cd, (s8) builtin_displaymethodstop, REG_ITMP1);
2755                                 x86_64_call_reg(cd, REG_ITMP1);
2756
2757                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2758                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2759
2760                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2761                         }
2762
2763                         /* restore saved registers                                        */
2764                         for (i = r->savintregcnt - 1; i >= r->maxsavintreguse; i--) {
2765                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, r->savintregs[i]);
2766                         }
2767                         for (i = r->savfltregcnt - 1; i >= r->maxsavfltreguse; i--) {
2768                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, r->savfltregs[i]);
2769                         }
2770
2771                         /* deallocate stack                                               */
2772                         if (parentargs_base) {
2773                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
2774                         }
2775
2776                         x86_64_ret(cd);
2777                         ALIGNCODENOP;
2778                         }
2779                         break;
2780
2781
2782                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
2783                         {
2784                                 s4 i, l, *s4ptr;
2785                                 void **tptr;
2786
2787                                 tptr = (void **) iptr->target;
2788
2789                                 s4ptr = iptr->val.a;
2790                                 l = s4ptr[1];                          /* low     */
2791                                 i = s4ptr[2];                          /* high    */
2792
2793                                 var_to_reg_int(s1, src, REG_ITMP1);
2794                                 M_INTMOVE(s1, REG_ITMP1);
2795                                 if (l != 0) {
2796                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
2797                                 }
2798                                 i = i - l + 1;
2799
2800                 /* range check */
2801                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
2802                                 x86_64_jcc(cd, X86_64_CC_A, 0);
2803
2804                 /* codegen_addreference(m, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
2805                                 codegen_addreference(m, (basicblock *) tptr[0], cd->mcodeptr);
2806
2807                                 /* build jump table top down and use address of lowest entry */
2808
2809                 /* s4ptr += 3 + i; */
2810                                 tptr += i;
2811
2812                                 while (--i >= 0) {
2813                                         /* dseg_addtarget(m, BlockPtrOfPC(*--s4ptr)); */
2814                                         dseg_addtarget(m, (basicblock *) tptr[0]);
2815                                         --tptr;
2816                                 }
2817
2818                                 /* length of dataseg after last dseg_addtarget is used by load */
2819
2820                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
2821                                 dseg_adddata(m, cd->mcodeptr);
2822                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
2823                                 x86_64_jmp_reg(cd, REG_ITMP1);
2824                                 ALIGNCODENOP;
2825                         }
2826                         break;
2827
2828
2829                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
2830                         {
2831                                 s4 i, l, val, *s4ptr;
2832                                 void **tptr;
2833
2834                                 tptr = (void **) iptr->target;
2835
2836                                 s4ptr = iptr->val.a;
2837                                 l = s4ptr[0];                          /* default  */
2838                                 i = s4ptr[1];                          /* count    */
2839
2840                                 MCODECHECK((i<<2)+8);
2841                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
2842                                 while (--i >= 0) {
2843                                         s4ptr += 2;
2844                                         ++tptr;
2845
2846                                         val = s4ptr[0];
2847                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
2848                                         x86_64_jcc(cd, X86_64_CC_E, 0);
2849                                         /* codegen_addreference(m, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
2850                                         codegen_addreference(m, (basicblock *) tptr[0], cd->mcodeptr);
2851                                 }
2852
2853                                 x86_64_jmp_imm(cd, 0);
2854                                 /* codegen_addreference(m, BlockPtrOfPC(l), cd->mcodeptr); */
2855
2856                                 tptr = (void **) iptr->target;
2857                                 codegen_addreference(m, (basicblock *) tptr[0], cd->mcodeptr);
2858
2859                                 ALIGNCODENOP;
2860                         }
2861                         break;
2862
2863
2864                 case ICMD_BUILTIN3:     /* ..., arg1, arg2, arg3 ==> ...              */
2865                                         /* op1 = return type, val.a = function pointer*/
2866                         s3 = 3;
2867                         goto gen_method;
2868
2869                 case ICMD_BUILTIN2:     /* ..., arg1, arg2 ==> ...                    */
2870                                         /* op1 = return type, val.a = function pointer*/
2871                         s3 = 2;
2872                         goto gen_method;
2873
2874                 case ICMD_BUILTIN1:     /* ..., arg1 ==> ...                          */
2875                                         /* op1 = return type, val.a = function pointer*/
2876                         s3 = 1;
2877                         goto gen_method;
2878
2879                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
2880                                         /* op1 = arg count, val.a = method pointer    */
2881
2882                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2883                                         /* op1 = arg count, val.a = method pointer    */
2884
2885                 case ICMD_INVOKEVIRTUAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
2886                                         /* op1 = arg count, val.a = method pointer    */
2887
2888                 case ICMD_INVOKEINTERFACE:/*.., objectref, [arg1, [arg2 ...]] ==> ... */
2889                                         /* op1 = arg count, val.a = method pointer    */
2890
2891                         s3 = iptr->op1;
2892
2893 gen_method: {
2894                         methodinfo   *lm;
2895                         classinfo    *ci;
2896                         stackptr      tmpsrc;
2897                         s4 iarg, farg;
2898
2899                         MCODECHECK((s3 << 1) + 64);
2900
2901                         tmpsrc = src;
2902                         s2 = s3;
2903                         iarg = 0;
2904                         farg = 0;
2905
2906                         /* copy arguments to registers or stack location                  */
2907                         for (; --s3 >= 0; src = src->prev) {
2908                                 IS_INT_LNG_TYPE(src->type) ? iarg++ : farg++;
2909                         }
2910
2911                         src = tmpsrc;
2912                         s3 = s2;
2913
2914                         s2 = (iarg > INT_ARG_CNT) ? iarg - INT_ARG_CNT : 0 + (farg > FLT_ARG_CNT) ? farg - FLT_ARG_CNT : 0;
2915
2916                         for (; --s3 >= 0; src = src->prev) {
2917                                 IS_INT_LNG_TYPE(src->type) ? iarg-- : farg--;
2918                                 if (src->varkind == ARGVAR) {
2919                                         if (IS_INT_LNG_TYPE(src->type)) {
2920                                                 if (iarg >= INT_ARG_CNT) {
2921                                                         s2--;
2922                                                 }
2923                                         } else {
2924                                                 if (farg >= FLT_ARG_CNT) {
2925                                                         s2--;
2926                                                 }
2927                                         }
2928                                         continue;
2929                                 }
2930
2931                                 if (IS_INT_LNG_TYPE(src->type)) {
2932                                         if (iarg < INT_ARG_CNT) {
2933                                                 s1 = r->argintregs[iarg];
2934                                                 var_to_reg_int(d, src, s1);
2935                                                 M_INTMOVE(d, s1);
2936
2937                                         } else {
2938                                                 var_to_reg_int(d, src, REG_ITMP1);
2939                                                 s2--;
2940                                                 x86_64_mov_reg_membase(cd, d, REG_SP, s2 * 8);
2941                                         }
2942
2943                                 } else {
2944                                         if (farg < FLT_ARG_CNT) {
2945                                                 s1 = r->argfltregs[farg];
2946                                                 var_to_reg_flt(d, src, s1);
2947                                                 M_FLTMOVE(d, s1);
2948
2949                                         } else {
2950                                                 var_to_reg_flt(d, src, REG_FTMP1);
2951                                                 s2--;
2952                                                 x86_64_movq_reg_membase(cd, d, REG_SP, s2 * 8);
2953                                         }
2954                                 }
2955                         } /* end of for */
2956
2957                         lm = iptr->val.a;
2958                         switch (iptr->opc) {
2959                                 case ICMD_BUILTIN3:
2960                                 case ICMD_BUILTIN2:
2961                                 case ICMD_BUILTIN1:
2962
2963                                         a = (s8) lm;
2964                                         d = iptr->op1;
2965
2966                                         x86_64_mov_imm_reg(cd, a, REG_ITMP1);
2967                                         x86_64_call_reg(cd, REG_ITMP1);
2968                                         break;
2969
2970                                 case ICMD_INVOKESTATIC:
2971
2972                                         a = (s8) lm->stubroutine;
2973                                         d = lm->returntype;
2974
2975                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
2976                                         x86_64_call_reg(cd, REG_ITMP2);
2977                                         break;
2978
2979                                 case ICMD_INVOKESPECIAL:
2980
2981                                         a = (s8) lm->stubroutine;
2982                                         d = lm->returntype;
2983
2984                                         gen_nullptr_check(r->argintregs[0]);    /* first argument contains pointer */
2985                                         x86_64_mov_membase_reg(cd, r->argintregs[0], 0, REG_ITMP2); /* access memory for hardware nullptr */
2986                                         x86_64_mov_imm_reg(cd, a, REG_ITMP2);
2987                                         x86_64_call_reg(cd, REG_ITMP2);
2988                                         break;
2989
2990                                 case ICMD_INVOKEVIRTUAL:
2991
2992                                         d = lm->returntype;
2993
2994                                         gen_nullptr_check(r->argintregs[0]);
2995                                         x86_64_mov_membase_reg(cd, r->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
2996                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, OFFSET(vftbl_t, table[0]) + sizeof(methodptr) * lm->vftblindex, REG_ITMP1);
2997                                         x86_64_call_reg(cd, REG_ITMP1);
2998                                         break;
2999
3000                                 case ICMD_INVOKEINTERFACE:
3001
3002                                         ci = lm->class;
3003                                         d = lm->returntype;
3004
3005                                         gen_nullptr_check(r->argintregs[0]);
3006                                         x86_64_mov_membase_reg(cd, r->argintregs[0], OFFSET(java_objectheader, vftbl), REG_ITMP2);
3007                                         x86_64_mov_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, interfacetable[0]) - sizeof(methodptr) * ci->index, REG_ITMP2);
3008                                         x86_64_mov_membase32_reg(cd, REG_ITMP2, sizeof(methodptr) * (lm - ci->methods), REG_ITMP1);
3009                                         x86_64_call_reg(cd, REG_ITMP1);
3010                                         break;
3011
3012                                 default:
3013                                         d = 0;
3014                                         error("Unkown ICMD-Command: %d", iptr->opc);
3015                                 }
3016
3017                         /* d contains return type */
3018
3019                         if (d != TYPE_VOID) {
3020                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3021                                         s1 = reg_of_var(m, iptr->dst, REG_RESULT);
3022                                         M_INTMOVE(REG_RESULT, s1);
3023                                         store_reg_to_var_int(iptr->dst, s1);
3024
3025                                 } else {
3026                                         s1 = reg_of_var(m, iptr->dst, REG_FRESULT);
3027                                         M_FLTMOVE(REG_FRESULT, s1);
3028                                         store_reg_to_var_flt(iptr->dst, s1);
3029                                 }
3030                         }
3031                         }
3032                         break;
3033
3034
3035                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3036
3037                                       /* op1:   0 == array, 1 == class                */
3038                                       /* val.a: (classinfo*) superclass               */
3039
3040 /*          superclass is an interface:
3041  *
3042  *          return (sub != NULL) &&
3043  *                 (sub->vftbl->interfacetablelength > super->index) &&
3044  *                 (sub->vftbl->interfacetable[-super->index] != NULL);
3045  *
3046  *          superclass is a class:
3047  *
3048  *          return ((sub != NULL) && (0
3049  *                  <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3050  *                  super->vftbl->diffvall));
3051  */
3052
3053                         {
3054                         classinfo *super = (classinfo*) iptr->val.a;
3055
3056 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3057             codegen_threadcritrestart(m, cd->mcodeptr - cd->mcodebase);
3058 #endif
3059
3060                         var_to_reg_int(s1, src, REG_ITMP1);
3061                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
3062                         if (s1 == d) {
3063                                 M_INTMOVE(s1, REG_ITMP1);
3064                                 s1 = REG_ITMP1;
3065                         }
3066                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3067                         if (iptr->op1) {                               /* class/interface */
3068                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3069                                         x86_64_test_reg_reg(cd, s1, s1);
3070
3071                                         /* TODO: clean up this calculation */
3072                                         a = 3;    /* mov_membase_reg */
3073                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3074
3075                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3076                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3077
3078                                         a += 3;    /* sub */
3079                                         CALCIMMEDIATEBYTES(a, super->index);
3080
3081                                         a += 3;    /* test */
3082
3083                                         a += 6;    /* jcc */
3084                                         a += 3;    /* mov_membase_reg */
3085                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3086
3087                                         a += 3;    /* test */
3088                                         a += 4;    /* setcc */
3089
3090                                         x86_64_jcc(cd, X86_64_CC_E, a);
3091
3092                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3093                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3094                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3095                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3096
3097                                         /* TODO: clean up this calculation */
3098                                         a = 0;
3099                                         a += 3;    /* mov_membase_reg */
3100                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3101
3102                                         a += 3;    /* test */
3103                                         a += 4;    /* setcc */
3104
3105                                         x86_64_jcc(cd, X86_64_CC_LE, a);
3106                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP1);
3107                                         x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3108                                         x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3109
3110                                 } else {                                   /* class           */
3111                                         x86_64_test_reg_reg(cd, s1, s1);
3112
3113                                         /* TODO: clean up this calculation */
3114                                         a = 3;    /* mov_membase_reg */
3115                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3116
3117                                         a += 10;   /* mov_imm_reg */
3118
3119                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3120                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3121
3122                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3123                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3124
3125                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3126                                         CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3127
3128                                         a += 3;    /* sub */
3129                                         a += 3;    /* xor */
3130                                         a += 3;    /* cmp */
3131                                         a += 4;    /* setcc */
3132
3133                                         x86_64_jcc(cd, X86_64_CC_E, a);
3134
3135                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3136                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3137 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3138                                         codegen_threadcritstart(m, cd->mcodeptr - cd->mcodebase);
3139 #endif
3140                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3141                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3142                                         x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3143 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3144                     codegen_threadcritstop(m, cd->mcodeptr - cd->mcodebase);
3145 #endif
3146                                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3147                                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3148                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3149                                         x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3150                                 }
3151                         }
3152                         else
3153                                 panic("internal error: no inlined array instanceof");
3154                         }
3155                         store_reg_to_var_int(iptr->dst, d);
3156                         break;
3157
3158                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3159
3160                                       /* op1:   0 == array, 1 == class                */
3161                                       /* val.a: (classinfo*) superclass               */
3162
3163 /*          superclass is an interface:
3164  *
3165  *          OK if ((sub == NULL) ||
3166  *                 (sub->vftbl->interfacetablelength > super->index) &&
3167  *                 (sub->vftbl->interfacetable[-super->index] != NULL));
3168  *
3169  *          superclass is a class:
3170  *
3171  *          OK if ((sub == NULL) || (0
3172  *                 <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3173  *                 super->vftbl->diffvall));
3174  */
3175
3176                         {
3177                         classinfo *super = (classinfo*) iptr->val.a;
3178
3179 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3180             codegen_threadcritrestart(m, cd->mcodeptr - cd->mcodebase);
3181 #endif
3182                         d = reg_of_var(m, iptr->dst, REG_ITMP3);
3183                         var_to_reg_int(s1, src, d);
3184                         if (iptr->op1) {                               /* class/interface */
3185                                 if (super->flags & ACC_INTERFACE) {        /* interface       */
3186                                         x86_64_test_reg_reg(cd, s1, s1);
3187
3188                                         /* TODO: clean up this calculation */
3189                                         a = 3;    /* mov_membase_reg */
3190                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3191
3192                                         a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3193                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength));
3194
3195                                         a += 3;    /* sub */
3196                                         CALCIMMEDIATEBYTES(a, super->index);
3197
3198                                         a += 3;    /* test */
3199                                         a += 6;    /* jcc */
3200
3201                                         a += 3;    /* mov_membase_reg */
3202                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*));
3203
3204                                         a += 3;    /* test */
3205                                         a += 6;    /* jcc */
3206
3207                                         x86_64_jcc(cd, X86_64_CC_E, a);
3208
3209                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3210                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetablelength), REG_ITMP2);
3211                                         x86_64_alu_imm_reg(cd, X86_64_SUB, super->index, REG_ITMP2);
3212                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3213                                         x86_64_jcc(cd, X86_64_CC_LE, 0);
3214                                         codegen_addxcastrefs(m, cd->mcodeptr);
3215                                         x86_64_mov_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, interfacetable[0]) - super->index * sizeof(methodptr*), REG_ITMP2);
3216                                         x86_64_test_reg_reg(cd, REG_ITMP2, REG_ITMP2);
3217                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3218                                         codegen_addxcastrefs(m, cd->mcodeptr);
3219
3220                                 } else {                                     /* class           */
3221                                         x86_64_test_reg_reg(cd, s1, s1);
3222
3223                                         /* TODO: clean up this calculation */
3224                                         a = 3;    /* mov_membase_reg */
3225                                         CALCOFFSETBYTES(a, s1, OFFSET(java_objectheader, vftbl));
3226                                         a += 10;   /* mov_imm_reg */
3227                                         a += 2;    /* movl_membase_reg - only if REG_ITMP1 == RAX */
3228                                         CALCOFFSETBYTES(a, REG_ITMP1, OFFSET(vftbl_t, baseval));
3229
3230                                         if (d != REG_ITMP3) {
3231                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3232                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3233                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3234                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3235                                                 a += 3;    /* sub */
3236
3237                                         } else {
3238                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3239                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, baseval));
3240                                                 a += 3;    /* sub */
3241                                                 a += 10;   /* mov_imm_reg */
3242                                                 a += 3;    /* movl_membase_reg - only if REG_ITMP2 == R10 */
3243                                                 CALCOFFSETBYTES(a, REG_ITMP2, OFFSET(vftbl_t, diffval));
3244                                         }
3245
3246                                         a += 3;    /* cmp */
3247                                         a += 6;    /* jcc */
3248
3249                                         x86_64_jcc(cd, X86_64_CC_E, a);
3250
3251                                         x86_64_mov_membase_reg(cd, s1, OFFSET(java_objectheader, vftbl), REG_ITMP1);
3252                                         x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3253 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3254                     codegen_threadcritstart(m, cd->mcodeptr - cd->mcodebase);
3255 #endif
3256                                         x86_64_movl_membase_reg(cd, REG_ITMP1, OFFSET(vftbl_t, baseval), REG_ITMP1);
3257                                         if (d != REG_ITMP3) {
3258                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP3);
3259                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3260 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3261                         codegen_threadcritstop(m, cd->mcodeptr - cd->mcodebase);
3262 #endif
3263                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP1);
3264
3265                                         } else {
3266                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, baseval), REG_ITMP2);
3267                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3268                                                 x86_64_mov_imm_reg(cd, (s8) super->vftbl, REG_ITMP2);
3269                                                 x86_64_movl_membase_reg(cd, REG_ITMP2, OFFSET(vftbl_t, diffval), REG_ITMP2);
3270 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3271                         codegen_threadcritstop(m, cd->mcodeptr - cd->mcodebase);
3272 #endif
3273                                         }
3274                                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
3275                                         x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3276                                         codegen_addxcastrefs(m, cd->mcodeptr);
3277                                 }
3278
3279                         } else
3280                                 panic("internal error: no inlined array checkcast");
3281                         }
3282                         M_INTMOVE(s1, d);
3283                         store_reg_to_var_int(iptr->dst, d);
3284                         break;
3285
3286                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3287
3288                         if (src->flags & INMEMORY) {
3289                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3290
3291                         } else {
3292                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3293                         }
3294                         x86_64_jcc(cd, X86_64_CC_L, 0);
3295                         codegen_addxcheckarefs(m, cd->mcodeptr);
3296                         break;
3297
3298                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3299
3300                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3301                         x86_64_jcc(cd, X86_64_CC_E, 0);
3302                         codegen_addxexceptionrefs(m, cd->mcodeptr);
3303                         break;
3304
3305                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3306                                          /* op1 = dimension, val.a = array descriptor */
3307
3308                         /* check for negative sizes and copy sizes to stack if necessary  */
3309
3310                         MCODECHECK((iptr->op1 << 1) + 64);
3311
3312                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3313                                 var_to_reg_int(s2, src, REG_ITMP1);
3314                                 x86_64_testl_reg_reg(cd, s2, s2);
3315                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3316                                 codegen_addxcheckarefs(m, cd->mcodeptr);
3317
3318                                 /* copy sizes to stack (argument numbers >= INT_ARG_CNT)      */
3319
3320                                 if (src->varkind != ARGVAR) {
3321                                         x86_64_mov_reg_membase(cd, s2, REG_SP, (s1 + INT_ARG_CNT) * 8);
3322                                 }
3323                         }
3324
3325                         /* a0 = dimension count */
3326                         x86_64_mov_imm_reg(cd, iptr->op1, r->argintregs[0]);
3327
3328                         /* a1 = arraydescriptor */
3329                         x86_64_mov_imm_reg(cd, (s8) iptr->val.a, r->argintregs[1]);
3330
3331                         /* a2 = pointer to dimensions = stack pointer */
3332                         x86_64_mov_reg_reg(cd, REG_SP, r->argintregs[2]);
3333
3334                         x86_64_mov_imm_reg(cd, (s8) builtin_nmultianewarray, REG_ITMP1);
3335                         x86_64_call_reg(cd, REG_ITMP1);
3336
3337                         s1 = reg_of_var(m, iptr->dst, REG_RESULT);
3338                         M_INTMOVE(REG_RESULT, s1);
3339                         store_reg_to_var_int(iptr->dst, s1);
3340                         break;
3341
3342                 default: error("Unknown pseudo command: %d", iptr->opc);
3343         } /* switch */
3344
3345         } /* for instruction */
3346
3347         /* copy values to interface registers */
3348
3349         src = bptr->outstack;
3350         len = bptr->outdepth;
3351         MCODECHECK(64 + len);
3352         while (src) {
3353                 len--;
3354                 if ((src->varkind != STACKVAR)) {
3355                         s2 = src->type;
3356                         if (IS_FLT_DBL_TYPE(s2)) {
3357                                 var_to_reg_flt(s1, src, REG_FTMP1);
3358                                 if (!(r->interfaces[len][s2].flags & INMEMORY)) {
3359                                         M_FLTMOVE(s1, r->interfaces[len][s2].regoff);
3360
3361                                 } else {
3362                                         x86_64_movq_reg_membase(cd, s1, REG_SP, r->interfaces[len][s2].regoff * 8);
3363                                 }
3364
3365                         } else {
3366                                 var_to_reg_int(s1, src, REG_ITMP1);
3367                                 if (!(r->interfaces[len][s2].flags & INMEMORY)) {
3368                                         M_INTMOVE(s1, r->interfaces[len][s2].regoff);
3369
3370                                 } else {
3371                                         x86_64_mov_reg_membase(cd, s1, REG_SP, r->interfaces[len][s2].regoff * 8);
3372                                 }
3373                         }
3374                 }
3375                 src = src->prev;
3376         }
3377         } /* if (bptr -> flags >= BBREACHED) */
3378         } /* for basic block */
3379
3380         {
3381
3382         /* generate bound check stubs */
3383
3384         u1 *xcodeptr = NULL;
3385         branchref *bref;
3386
3387         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3388                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3389                                   bref->branchpos,
3390                                                   cd->mcodeptr - cd->mcodebase);
3391
3392                 MCODECHECK(50);
3393
3394                 /* move index register into REG_ITMP1 */
3395                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);              /* 3 bytes  */
3396
3397                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3398                 dseg_adddata(m, cd->mcodeptr);
3399                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3400                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3401
3402                 if (xcodeptr != NULL) {
3403                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3404
3405                 } else {
3406                         xcodeptr = cd->mcodeptr;
3407
3408                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3409                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3410                         x86_64_mov_imm_reg(cd, (s8) string_java_lang_ArrayIndexOutOfBoundsException, r->argintregs[0]);
3411                         x86_64_mov_reg_reg(cd, REG_ITMP1, r->argintregs[1]);
3412                         x86_64_mov_imm_reg(cd, (s8) new_exception_int, REG_ITMP3);
3413                         x86_64_call_reg(cd, REG_ITMP3);
3414                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3415                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3416
3417                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3418                         x86_64_jmp_reg(cd, REG_ITMP3);
3419                 }
3420         }
3421
3422         /* generate negative array size check stubs */
3423
3424         xcodeptr = NULL;
3425
3426         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3427                 if ((m->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3428                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3429                                                           bref->branchpos,
3430                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3431                         continue;
3432                 }
3433
3434                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3435                                   bref->branchpos,
3436                                                   cd->mcodeptr - cd->mcodebase);
3437
3438                 MCODECHECK(50);
3439
3440                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                         /* 10 bytes */
3441                 dseg_adddata(m, cd->mcodeptr);
3442                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);    /* 10 bytes */
3443                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);     /* 3 bytes  */
3444
3445                 if (xcodeptr != NULL) {
3446                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3447
3448                 } else {
3449                         xcodeptr = cd->mcodeptr;
3450
3451                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3452                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3453                         x86_64_mov_imm_reg(cd, (s8) string_java_lang_NegativeArraySizeException, r->argintregs[0]);
3454                         x86_64_mov_imm_reg(cd, (s8) new_exception, REG_ITMP3);
3455                         x86_64_call_reg(cd, REG_ITMP3);
3456                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3457                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3458
3459                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3460                         x86_64_jmp_reg(cd, REG_ITMP3);
3461                 }
3462         }
3463
3464         /* generate cast check stubs */
3465
3466         xcodeptr = NULL;
3467
3468         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3469                 if ((m->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3470                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3471                                                           bref->branchpos,
3472                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3473                         continue;
3474                 }
3475
3476                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3477                                   bref->branchpos,
3478                                                   cd->mcodeptr - cd->mcodebase);
3479
3480                 MCODECHECK(50);
3481
3482                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3483                 dseg_adddata(m, cd->mcodeptr);
3484                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);     /* 10 bytes */
3485                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3486
3487                 if (xcodeptr != NULL) {
3488                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3489
3490                 } else {
3491                         xcodeptr = cd->mcodeptr;
3492
3493                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3494                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3495                         x86_64_mov_imm_reg(cd, (s8) string_java_lang_ClassCastException, r->argintregs[0]);
3496                         x86_64_mov_imm_reg(cd, (s8) new_exception, REG_ITMP3);
3497                         x86_64_call_reg(cd, REG_ITMP3);
3498                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3499                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3500
3501                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3502                         x86_64_jmp_reg(cd, REG_ITMP3);
3503                 }
3504         }
3505
3506         /* generate divide by zero check stubs */
3507
3508         xcodeptr = NULL;
3509
3510         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3511                 if ((m->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3512                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3513                                                           bref->branchpos,
3514                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3515                         continue;
3516                 }
3517
3518                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3519                                   bref->branchpos,
3520                                                   cd->mcodeptr - cd->mcodebase);
3521
3522                 MCODECHECK(50);
3523
3524                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3525                 dseg_adddata(m, cd->mcodeptr);
3526                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);      /* 10 bytes */
3527                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC);    /* 3 bytes  */
3528
3529                 if (xcodeptr != NULL) {
3530                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3531
3532                 } else {
3533                         xcodeptr = cd->mcodeptr;
3534
3535                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3536                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3537                         x86_64_mov_imm_reg(cd, (u8) string_java_lang_ArithmeticException, r->argintregs[0]);
3538                         x86_64_mov_imm_reg(cd, (u8) string_java_lang_ArithmeticException_message, r->argintregs[1]);
3539                         x86_64_mov_imm_reg(cd, (u8) new_exception, REG_ITMP3);
3540                         x86_64_call_reg(cd, REG_ITMP3);
3541                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3542                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3543
3544                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3545                         x86_64_jmp_reg(cd, REG_ITMP3);
3546                 }
3547         }
3548
3549         /* generate exception check stubs */
3550
3551         xcodeptr = NULL;
3552
3553         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
3554                 if ((m->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3555                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3556                                                           bref->branchpos,
3557                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3558                         continue;
3559                 }
3560
3561                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3562                                   bref->branchpos,
3563                                                   cd->mcodeptr - cd->mcodebase);
3564
3565                 MCODECHECK(50);
3566
3567                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3568                 dseg_adddata(m, cd->mcodeptr);
3569                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3570                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3571
3572                 if (xcodeptr != NULL) {
3573                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3574
3575                 } else {
3576                         xcodeptr = cd->mcodeptr;
3577
3578 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3579                         x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3580                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0);
3581                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
3582                         x86_64_call_reg(cd, REG_ITMP1);
3583                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3584                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3585                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3586                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
3587                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
3588 #else
3589                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
3590                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
3591                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
3592 #endif
3593
3594                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
3595                         x86_64_jmp_reg(cd, REG_ITMP3);
3596                 }
3597         }
3598
3599         /* generate null pointer check stubs */
3600
3601         xcodeptr = NULL;
3602
3603         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3604                 if ((m->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3605                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3606                                                           bref->branchpos,
3607                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
3608                         continue;
3609                 }
3610
3611                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3612                                   bref->branchpos,
3613                                                   cd->mcodeptr - cd->mcodebase);
3614
3615                 MCODECHECK(50);
3616
3617                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                        /* 10 bytes */
3618                 dseg_adddata(m, cd->mcodeptr);
3619                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);     /* 10 bytes */
3620                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC);    /* 3 bytes  */
3621
3622                 if (xcodeptr != NULL) {
3623                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3624
3625                 } else {
3626                         xcodeptr = cd->mcodeptr;
3627
3628                         x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3629                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 0 * 8);
3630                         x86_64_mov_imm_reg(cd, (s8) string_java_lang_NullPointerException, r->argintregs[0]);
3631                         x86_64_mov_imm_reg(cd, (s8) new_exception, REG_ITMP3);
3632                         x86_64_call_reg(cd, REG_ITMP3);
3633                         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_ITMP2_XPC);
3634                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3635
3636                         x86_64_mov_imm_reg(cd, (s8) asm_handle_exception, REG_ITMP3);
3637                         x86_64_jmp_reg(cd, REG_ITMP3);
3638                 }
3639         }
3640         }
3641
3642         codegen_finish(m, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
3643 }
3644
3645
3646 /* function createcompilerstub *************************************************
3647
3648         creates a stub routine which calls the compiler
3649
3650 *******************************************************************************/
3651
3652 #define COMPSTUBSIZE 23
3653
3654 u1 *createcompilerstub(methodinfo *m)
3655 {
3656         u1 *s = CNEW(u1, COMPSTUBSIZE);     /* memory to hold the stub            */
3657         codegendata *cd;
3658
3659         /* setup codegendata structure */
3660         codegen_setup(m);
3661
3662         cd = m->codegendata;
3663     cd->mcodeptr = s;
3664
3665         /* code for the stub */
3666         x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1); /* pass method pointer to compiler */
3667         x86_64_mov_imm_reg(cd, (s8) asm_call_jit_compiler, REG_ITMP3);/* load address */
3668         x86_64_jmp_reg(cd, REG_ITMP3);          /* jump to compiler                   */
3669
3670         /* free codegendata memory */
3671         codegen_close(m);
3672
3673 #if defined(STATISTICS)
3674         if (opt_stat)
3675                 count_cstub_len += COMPSTUBSIZE;
3676 #endif
3677
3678         return s;
3679 }
3680
3681
3682 /* function removecompilerstub *************************************************
3683
3684      deletes a compilerstub from memory  (simply by freeing it)
3685
3686 *******************************************************************************/
3687
3688 void removecompilerstub(u1 *stub)
3689 {
3690         CFREE(stub, COMPSTUBSIZE);
3691 }
3692
3693
3694 /* function: createnativestub **************************************************
3695
3696         creates a stub routine which calls a native method
3697
3698 *******************************************************************************/
3699
3700 /* #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3701 /* static java_objectheader **(*callgetexceptionptrptr)() = builtin_get_exceptionptrptr; */
3702 /* #endif */
3703
3704 #define NATIVESTUBSIZE 420
3705
3706 u1 *createnativestub(functionptr f, methodinfo *m)
3707 {
3708         u1 *s = CNEW(u1, NATIVESTUBSIZE);   /* memory to hold the stub            */
3709         s4 stackframesize;                  /* size of stackframe if needed       */
3710         registerdata *r;
3711         codegendata *cd;
3712
3713         /* setup codegendata structure */
3714         codegen_setup(m);
3715
3716         /* initialize registers before using it */
3717         reg_init(m);
3718
3719         /* keep code size smaller */
3720         r = m->registerdata;
3721         cd = m->codegendata;
3722
3723         cd->mcodeptr = s;
3724
3725     descriptor2types(m);                /* set paramcount and paramtypes      */
3726
3727         /* if function is static, check for initialized */
3728
3729         if (m->flags & ACC_STATIC) {
3730                 /* if class isn't yet initialized, do it */
3731                 if (!m->class->initialized) {
3732                         /* call helper function which patches this code */
3733                         x86_64_mov_imm_reg(cd, (u8) m->class, REG_ITMP1);
3734                         x86_64_mov_imm_reg(cd, (u8) asm_check_clinit, REG_ITMP2);
3735                         x86_64_call_reg(cd, REG_ITMP2);
3736                 }
3737         }
3738
3739         if (runverbose) {
3740                 s4 p, l, s1;
3741
3742                 x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3743
3744                 x86_64_mov_reg_membase(cd, r->argintregs[0], REG_SP, 1 * 8);
3745                 x86_64_mov_reg_membase(cd, r->argintregs[1], REG_SP, 2 * 8);
3746                 x86_64_mov_reg_membase(cd, r->argintregs[2], REG_SP, 3 * 8);
3747                 x86_64_mov_reg_membase(cd, r->argintregs[3], REG_SP, 4 * 8);
3748                 x86_64_mov_reg_membase(cd, r->argintregs[4], REG_SP, 5 * 8);
3749                 x86_64_mov_reg_membase(cd, r->argintregs[5], REG_SP, 6 * 8);
3750
3751                 x86_64_movq_reg_membase(cd, r->argfltregs[0], REG_SP, 7 * 8);
3752                 x86_64_movq_reg_membase(cd, r->argfltregs[1], REG_SP, 8 * 8);
3753                 x86_64_movq_reg_membase(cd, r->argfltregs[2], REG_SP, 9 * 8);
3754                 x86_64_movq_reg_membase(cd, r->argfltregs[3], REG_SP, 10 * 8);
3755 /*              x86_64_movq_reg_membase(cd, r->argfltregs[4], REG_SP, 11 * 8); */
3756 /*              x86_64_movq_reg_membase(cd, r->argfltregs[5], REG_SP, 12 * 8); */
3757 /*              x86_64_movq_reg_membase(cd, r->argfltregs[6], REG_SP, 13 * 8); */
3758 /*              x86_64_movq_reg_membase(cd, r->argfltregs[7], REG_SP, 14 * 8); */
3759
3760                 /* show integer hex code for float arguments */
3761                 for (p = 0, l = 0; p < m->paramcount; p++) {
3762                         if (IS_FLT_DBL_TYPE(m->paramtypes[p])) {
3763                                 for (s1 = (m->paramcount > INT_ARG_CNT) ? INT_ARG_CNT - 2 : m->paramcount - 2; s1 >= p; s1--) {
3764                                         x86_64_mov_reg_reg(cd, r->argintregs[s1], r->argintregs[s1 + 1]);
3765                                 }
3766
3767                                 x86_64_movd_freg_reg(cd, r->argfltregs[l], r->argintregs[p]);
3768                                 l++;
3769                         }
3770                 }
3771
3772                 x86_64_mov_imm_reg(cd, (s8) m, REG_ITMP1);
3773                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
3774                 x86_64_mov_imm_reg(cd, (s8) builtin_trace_args, REG_ITMP1);
3775                 x86_64_call_reg(cd, REG_ITMP1);
3776
3777                 x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, r->argintregs[0]);
3778                 x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, r->argintregs[1]);
3779                 x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, r->argintregs[2]);
3780                 x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, r->argintregs[3]);
3781                 x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, r->argintregs[4]);
3782                 x86_64_mov_membase_reg(cd, REG_SP, 6 * 8, r->argintregs[5]);
3783
3784                 x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, r->argfltregs[0]);
3785                 x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, r->argfltregs[1]);
3786                 x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, r->argfltregs[2]);
3787                 x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, r->argfltregs[3]);
3788 /*              x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, r->argfltregs[4]); */
3789 /*              x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, r->argfltregs[5]); */
3790 /*              x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, r->argfltregs[6]); */
3791 /*              x86_64_movq_membase_reg(cd, REG_SP, 14 * 8, r->argfltregs[7]); */
3792
3793                 x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
3794         }
3795
3796 #if 0
3797         x86_64_alu_imm_reg(cd, X86_64_SUB, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3798
3799         /* save callee saved float registers */
3800         x86_64_movq_reg_membase(cd, XMM15, REG_SP, 0 * 8);
3801         x86_64_movq_reg_membase(cd, XMM14, REG_SP, 1 * 8);
3802         x86_64_movq_reg_membase(cd, XMM13, REG_SP, 2 * 8);
3803         x86_64_movq_reg_membase(cd, XMM12, REG_SP, 3 * 8);
3804         x86_64_movq_reg_membase(cd, XMM11, REG_SP, 4 * 8);
3805         x86_64_movq_reg_membase(cd, XMM10, REG_SP, 5 * 8);
3806 #endif
3807
3808         /* save argument registers on stack -- if we have to */
3809         if ((m->flags & ACC_STATIC && m->paramcount > (INT_ARG_CNT - 2)) || m->paramcount > (INT_ARG_CNT - 1)) {
3810                 s4 i;
3811                 s4 paramshiftcnt = (m->flags & ACC_STATIC) ? 2 : 1;
3812                 s4 stackparamcnt = (m->paramcount > INT_ARG_CNT) ? m->paramcount - INT_ARG_CNT : 0;
3813
3814                 stackframesize = stackparamcnt + paramshiftcnt;
3815
3816                 /* keep stack 16-byte aligned */
3817                 if ((stackframesize % 2) == 0) stackframesize++;
3818
3819                 x86_64_alu_imm_reg(cd, X86_64_SUB, stackframesize * 8, REG_SP);
3820
3821                 /* copy stack arguments into new stack frame -- if any */
3822                 for (i = 0; i < stackparamcnt; i++) {
3823                         x86_64_mov_membase_reg(cd, REG_SP, (stackparamcnt + 1 + i) * 8, REG_ITMP1);
3824                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, (paramshiftcnt + i) * 8);
3825                 }
3826
3827                 if (m->flags & ACC_STATIC) {
3828                         x86_64_mov_reg_membase(cd, r->argintregs[5], REG_SP, 1 * 8);
3829                         x86_64_mov_reg_membase(cd, r->argintregs[4], REG_SP, 0 * 8);
3830
3831                 } else {
3832                         x86_64_mov_reg_membase(cd, r->argintregs[5], REG_SP, 0 * 8);
3833                 }
3834
3835         } else {
3836                 /* keep stack 16-byte aligned -- this is essential for x86_64 */
3837                 x86_64_alu_imm_reg(cd, X86_64_SUB, 8, REG_SP);
3838                 stackframesize = 1;
3839         }
3840
3841         if (m->flags & ACC_STATIC) {
3842                 x86_64_mov_reg_reg(cd, r->argintregs[3], r->argintregs[5]);
3843                 x86_64_mov_reg_reg(cd, r->argintregs[2], r->argintregs[4]);
3844                 x86_64_mov_reg_reg(cd, r->argintregs[1], r->argintregs[3]);
3845                 x86_64_mov_reg_reg(cd, r->argintregs[0], r->argintregs[2]);
3846
3847                 /* put class into second argument register */
3848                 x86_64_mov_imm_reg(cd, (u8) m->class, r->argintregs[1]);
3849
3850         } else {
3851                 x86_64_mov_reg_reg(cd, r->argintregs[4], r->argintregs[5]);
3852                 x86_64_mov_reg_reg(cd, r->argintregs[3], r->argintregs[4]);
3853                 x86_64_mov_reg_reg(cd, r->argintregs[2], r->argintregs[3]);
3854                 x86_64_mov_reg_reg(cd, r->argintregs[1], r->argintregs[2]);
3855                 x86_64_mov_reg_reg(cd, r->argintregs[0], r->argintregs[1]);
3856         }
3857
3858         /* put env into first argument register */
3859         x86_64_mov_imm_reg(cd, (u8) &env, r->argintregs[0]);
3860
3861         x86_64_mov_imm_reg(cd, (u8) f, REG_ITMP1);
3862         x86_64_call_reg(cd, REG_ITMP1);
3863
3864         /* remove stackframe if there is one */
3865         if (stackframesize) {
3866                 x86_64_alu_imm_reg(cd, X86_64_ADD, stackframesize * 8, REG_SP);
3867         }
3868
3869         if (runverbose) {
3870                 x86_64_alu_imm_reg(cd, X86_64_SUB, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
3871
3872                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
3873                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
3874
3875                 x86_64_mov_imm_reg(cd, (u8) m, r->argintregs[0]);
3876                 x86_64_mov_reg_reg(cd, REG_RESULT, r->argintregs[1]);
3877                 M_FLTMOVE(REG_FRESULT, r->argfltregs[0]);
3878                 M_FLTMOVE(REG_FRESULT, r->argfltregs[1]);
3879
3880                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
3881                 x86_64_call_reg(cd, REG_ITMP1);
3882
3883                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
3884                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
3885
3886                 x86_64_alu_imm_reg(cd, X86_64_ADD, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
3887         }
3888
3889 #if 0
3890         /* restore callee saved registers */
3891         x86_64_movq_membase_reg(cd, REG_SP, 0 * 8, XMM15);
3892         x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, XMM14);
3893         x86_64_movq_membase_reg(cd, REG_SP, 2 * 8, XMM13);
3894         x86_64_movq_membase_reg(cd, REG_SP, 3 * 8, XMM12);
3895         x86_64_movq_membase_reg(cd, REG_SP, 4 * 8, XMM11);
3896         x86_64_movq_membase_reg(cd, REG_SP, 5 * 8, XMM10);
3897
3898         x86_64_alu_imm_reg(cd, X86_64_ADD, 7 * 8, REG_SP);    /* keep stack 16-byte aligned */
3899 #endif
3900
3901 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3902         x86_64_push_reg(cd, REG_RESULT);
3903 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
3904         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
3905         x86_64_call_reg(cd, REG_ITMP3);
3906         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
3907         x86_64_pop_reg(cd, REG_RESULT);
3908 #else
3909         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
3910         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
3911 #endif
3912         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3913         x86_64_jcc(cd, X86_64_CC_NE, 1);
3914
3915         x86_64_ret(cd);
3916
3917 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3918         x86_64_push_reg(cd, REG_ITMP3);
3919 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
3920         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
3921         x86_64_call_reg(cd, REG_ITMP3);
3922         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
3923         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
3924 #else
3925         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
3926         x86_64_mov_imm_reg(cd, (s8) &_exceptionptr, REG_ITMP3);
3927         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
3928         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
3929 #endif
3930
3931         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
3932         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
3933
3934         x86_64_mov_imm_reg(cd, (s8) asm_handle_nat_exception, REG_ITMP3);
3935         x86_64_jmp_reg(cd, REG_ITMP3);
3936
3937 #if 0
3938         {
3939                 static int stubprinted;
3940                 if (!stubprinted)
3941                         printf("stubsize: %d\n", ((long) cd->mcodeptr - (long) s));
3942                 stubprinted = 1;
3943         }
3944 #endif
3945
3946         /* free codegendata memory */
3947         codegen_close(m);
3948
3949 #if defined(STATISTICS)
3950         if (opt_stat)
3951                 count_nstub_len += NATIVESTUBSIZE;
3952 #endif
3953
3954         return s;
3955 }
3956
3957
3958 /* function: removenativestub **************************************************
3959
3960     removes a previously created native-stub from memory
3961
3962 *******************************************************************************/
3963
3964 void removenativestub(u1 *stub)
3965 {
3966         CFREE(stub, NATIVESTUBSIZE);
3967 }
3968
3969
3970 /*
3971  * These are local overrides for various environment variables in Emacs.
3972  * Please do not remove this and leave it at the end of the file, where
3973  * Emacs will automagically detect them.
3974  * ---------------------------------------------------------------------
3975  * Local variables:
3976  * mode: c
3977  * indent-tabs-mode: t
3978  * c-basic-offset: 4
3979  * tab-width: 4
3980  * End:
3981  */