src/vm/jit/x86_64/codegen.c

   1 /* vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    $Id: codegen.c 2442 2005-05-11 12:05:08Z twisti $
  31
  32 */
  33
  34
  35 #define _GNU_SOURCE
  36
  37 #include <stdio.h>
  38 #include <ucontext.h>
  39
  40 #include "cacao/cacao.h"
  41 #include "native/native.h"
  42 #include "vm/global.h"
  43 #include "vm/builtin.h"
  44 #include "vm/loader.h"
  45 #include "vm/tables.h"
  46 #include "vm/jit/asmpart.h"
  47 #include "vm/jit/jit.h"
  48 #include "vm/jit/reg.h"
  49 #include "vm/jit/parse.h"
  50 #include "vm/jit/patcher.h"
  51 #include "vm/jit/x86_64/arch.h"
  52 #include "vm/jit/x86_64/codegen.h"
  53 #include "vm/jit/x86_64/emitfuncs.h"
  54 #include "vm/jit/x86_64/types.h"
  55 #include "vm/jit/x86_64/asmoffsets.h"
  56 #include "vm/jit/helper.h"
  57 #include "vm/statistics.h"
  58
  59 /* register descripton - array ************************************************/
  60
  61 /* #define REG_RES   0         reserved register for OS or code generator     */
  62 /* #define REG_RET   1         return value register                          */
  63 /* #define REG_EXC   2         exception value register (only old jit)        */
  64 /* #define REG_SAV   3         (callee) saved register                        */
  65 /* #define REG_TMP   4         scratch temporary register (caller saved)      */
  66 /* #define REG_ARG   5         argument register (caller saved)               */
  67
  68 /* #define REG_END   -1        last entry in tables                           */
  69
  70 static int nregdescint[] = {
  71     REG_RET, REG_ARG, REG_ARG, REG_TMP, REG_RES, REG_SAV, REG_ARG, REG_ARG,
  72     REG_ARG, REG_ARG, REG_RES, REG_RES, REG_SAV, REG_SAV, REG_SAV, REG_SAV,
  73     REG_END
  74 };
  75
  76
  77 static int nregdescfloat[] = {
  78     REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG, REG_ARG,
  79     REG_RES, REG_RES, REG_RES, REG_TMP, REG_TMP, REG_TMP, REG_TMP, REG_TMP,
  80     REG_END
  81 };
  82
  83
  84 /* Include independent code generation stuff -- include after register        */
  85 /* descriptions to avoid extern definitions.                                  */
  86
  87 #include "vm/jit/codegen.inc"
  88 #include "vm/jit/reg.inc"
  89 #ifdef LSRA
  90 #include "vm/jit/lsra.inc"
  91 #endif
  92
  93 #if 0
  94 #define JWNATIVEDEBUG(x) true
  95 #else
  96 #define JWNATIVEDEBUG(x) x
  97 #endif
  98 void codegen_dummy_func() { log_text("codegen_dummy_func"); }
  99
 100 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
 101 void thread_restartcriticalsection(ucontext_t *uc)
 102 {
 103         void *critical;
 104
 105         critical = thread_checkcritical((void *) uc->uc_mcontext.gregs[REG_RIP]);
 106
 107         if (critical)
 108                 uc->uc_mcontext.gregs[REG_RIP] = (u8) critical;
 109 }
 110 #endif
 111
 112
 113 /* NullPointerException signal handler for hardware null pointer check */
 114
 115 void catch_NullPointerException(int sig, siginfo_t *siginfo, void *_p)
 116 {
 117         sigset_t nsig;
 118
 119         struct ucontext *_uc = (struct ucontext *) _p;
 120         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 121         struct sigaction act;
 122         java_objectheader *xptr;
 123
 124         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 125
 126         act.sa_sigaction = catch_NullPointerException;       /* reinstall handler */
 127         act.sa_flags = SA_SIGINFO;
 128         sigaction(sig, &act, NULL);
 129
 130         sigemptyset(&nsig);
 131         sigaddset(&nsig, sig);
 132         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 133
 134 #if 0
 135         xptr = new_nullpointerexception();
 136
 137         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 138         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 139         sigctx->rip = (u8) asm_handle_exception;
 140 #endif
 141
 142         sigctx->rax = (u8) string_java_lang_NullPointerException;
 143         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 144         sigctx->rip = (u8) asm_throw_and_handle_exception;
 145
 146         return;
 147 }
 148
 149
 150 /* ArithmeticException signal handler for hardware divide by zero check */
 151
 152 void catch_ArithmeticException(int sig, siginfo_t *siginfo, void *_p)
 153 {
 154         sigset_t nsig;
 155
 156         struct ucontext *_uc = (struct ucontext *) _p;
 157         struct sigcontext *sigctx = (struct sigcontext *) &_uc->uc_mcontext;
 158         struct sigaction act;
 159         java_objectheader *xptr;
 160
 161         /* Reset signal handler - necessary for SysV, does no harm for BSD */
 162
 163         act.sa_sigaction = catch_ArithmeticException;        /* reinstall handler */
 164         act.sa_flags = SA_SIGINFO;
 165         sigaction(sig, &act, NULL);
 166
 167         sigemptyset(&nsig);
 168         sigaddset(&nsig, sig);
 169         sigprocmask(SIG_UNBLOCK, &nsig, NULL);               /* unblock signal    */
 170
 171 #if 0
 172         xptr = new_arithmeticexception();
 173
 174         sigctx->rax = (u8) xptr;                             /* REG_ITMP1_XPTR    */
 175 #endif
 176         sigctx->r10 = sigctx->rip;                           /* REG_ITMP2_XPC     */
 177         sigctx->rip = (u8) asm_throw_and_handle_hardware_arithmetic_exception;
 178
 179         return;
 180 }
 181
 182
 183 void init_exceptions(void)
 184 {
 185         struct sigaction act;
 186
 187         /* install signal handlers we need to convert to exceptions */
 188         sigemptyset(&act.sa_mask);
 189
 190         if (!checknull) {
 191 #if defined(SIGSEGV)
 192                 act.sa_sigaction = catch_NullPointerException;
 193                 act.sa_flags = SA_SIGINFO;
 194                 sigaction(SIGSEGV, &act, NULL);
 195 #endif
 196
 197 #if defined(SIGBUS)
 198                 act.sa_sigaction = catch_NullPointerException;
 199                 act.sa_flags = SA_SIGINFO;
 200                 sigaction(SIGBUS, &act, NULL);
 201 #endif
 202         }
 203
 204         act.sa_sigaction = catch_ArithmeticException;
 205         act.sa_flags = SA_SIGINFO;
 206         sigaction(SIGFPE, &act, NULL);
 207 }
 208
 209
 210 /* function gen_mcode **********************************************************
 211
 212         generates machine code
 213
 214 *******************************************************************************/
 215
 216 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
 217 {
 218         s4              len, s1, s2, s3, d;
 219         u2              currentline;
 220         ptrint          a;
 221         s4              parentargs_base;
 222         stackptr        src;
 223         varinfo        *var;
 224         basicblock     *bptr;
 225         instruction    *iptr;
 226         exceptiontable *ex;
 227
 228         {
 229         s4 i, p, pa, t, l;
 230         s4 savedregs_num;
 231
 232         savedregs_num = 0;
 233
 234         /* space to save used callee saved registers */
 235
 236         savedregs_num += (rd->savintregcnt - rd->maxsavintreguse);
 237         savedregs_num += (rd->savfltregcnt - rd->maxsavfltreguse);
 238
 239         parentargs_base = rd->maxmemuse + savedregs_num;
 240
 241 #if defined(USE_THREADS)           /* space to save argument of monitor_enter */
 242
 243         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 244                 parentargs_base++;
 245
 246 #endif
 247
 248     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 249         /* code e.g. libc or jni (alignment problems with movaps).                */
 250
 251         if (!m->isleafmethod || runverbose)
 252                 parentargs_base |= 0x1;
 253
 254         /* create method header */
 255
 256         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 257         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 258
 259 #if defined(USE_THREADS)
 260
 261         /* IsSync contains the offset relative to the stack pointer for the
 262            argument of monitor_exit used in the exception handler. Since the
 263            offset could be zero and give a wrong meaning of the flag it is
 264            offset by one.
 265         */
 266
 267         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 268                 (void) dseg_adds4(cd, (rd->maxmemuse + 1) * 8);     /* IsSync         */
 269         else
 270
 271 #endif
 272
 273                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 274
 275         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 276         (void) dseg_adds4(cd, rd->savintregcnt - rd->maxsavintreguse);/* IntSave  */
 277         (void) dseg_adds4(cd, rd->savfltregcnt - rd->maxsavfltreguse);/* FltSave  */
 278
 279         (void) dseg_addlinenumbertablesize(cd);
 280
 281         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 282
 283         /* create exception table */
 284
 285         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 286                 dseg_addtarget(cd, ex->start);
 287                 dseg_addtarget(cd, ex->end);
 288                 dseg_addtarget(cd, ex->handler);
 289                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 290         }
 291
 292         /* initialize mcode variables */
 293
 294         cd->mcodeptr = (u1 *) cd->mcodebase;
 295         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 296         MCODECHECK(128 + m->paramcount);
 297
 298         /* create stack frame (if necessary) */
 299
 300         if (parentargs_base) {
 301                 x86_64_alu_imm_reg(cd, X86_64_SUB, parentargs_base * 8, REG_SP);
 302         }
 303
 304         /* save used callee saved registers */
 305
 306         p = parentargs_base;
 307         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
 308                 p--; x86_64_mov_reg_membase(cd, rd->savintregs[i], REG_SP, p * 8);
 309         }
 310         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
 311                 p--; x86_64_movq_reg_membase(cd, rd->savfltregs[i], REG_SP, p * 8);
 312         }
 313
 314         /* take arguments out of register or stack frame */
 315
 316         for (p = 0, l = 0, s1 = 0, s2 = 0; p < m->paramcount; p++) {
 317                 t = m->paramtypes[p];
 318                 var = &(rd->locals[l][t]);
 319                 l++;
 320                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 321                         l++;
 322                 if (var->type < 0) {
 323                         if (IS_INT_LNG_TYPE(t))
 324                                 s1++;
 325                         else
 326                                 s2++;
 327                         continue;
 328                 }
 329                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 330                         if (s1 < INT_ARG_CNT) {                  /* register arguments    */
 331                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 332                                         M_INTMOVE(rd->argintregs[s1], var->regoff);
 333
 334                                 } else {                             /* reg arg -> spilled    */
 335                                     x86_64_mov_reg_membase(cd, rd->argintregs[s1], REG_SP, var->regoff * 8);
 336                                 }
 337
 338                         } else {                                 /* stack arguments       */
 339                                 pa = s1 - INT_ARG_CNT;
 340                                 if (s2 >= FLT_ARG_CNT) {
 341                                         pa += s2 - FLT_ARG_CNT;
 342                                 }
 343                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 344                                         x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);    /* + 8 for return address */
 345                                 } else {                             /* stack arg -> spilled  */
 346                                         /* x86_64_mov_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_ITMP1); */    /* + 8 for return address */
 347 /*                                      x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, var->regoff * 8); */
 348                                         var->regoff = parentargs_base + pa + 1;
 349                                 }
 350                         }
 351                         s1++;
 352
 353                 } else {                                     /* floating args         */
 354                         if (s2 < FLT_ARG_CNT) {                  /* register arguments    */
 355                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 356                                         M_FLTMOVE(rd->argfltregs[s2], var->regoff);
 357
 358                                 } else {                                         /* reg arg -> spilled    */
 359                                         x86_64_movq_reg_membase(cd, rd->argfltregs[s2], REG_SP, var->regoff * 8);
 360                                 }
 361
 362                         } else {                                 /* stack arguments       */
 363                                 pa = s2 - FLT_ARG_CNT;
 364                                 if (s1 >= INT_ARG_CNT) {
 365                                         pa += s1 - INT_ARG_CNT;
 366                                 }
 367                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 368                                         x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, var->regoff);
 369
 370                                 } else {
 371 /*                                      x86_64_movq_membase_reg(cd, REG_SP, (parentargs_base + pa) * 8 + 8, REG_FTMP1); */
 372 /*                                      x86_64_movq_reg_membase(cd, REG_FTMP1, REG_SP, var->regoff * 8); */
 373                                         var->regoff = parentargs_base + pa + 1;
 374                                 }
 375                         }
 376                         s2++;
 377                 }
 378         }  /* end for */
 379
 380         /* save monitorenter argument */
 381
 382 #if defined(USE_THREADS)
 383         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 384                 /* stack offset for monitor argument */
 385
 386                 s1 = rd->maxmemuse;
 387
 388                 if (runverbose) {
 389                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT) * 8, REG_SP);
 390
 391                         for (p = 0; p < INT_ARG_CNT; p++)
 392                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 393
 394                         for (p = 0; p < FLT_ARG_CNT; p++)
 395                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 396
 397                         if (m->isleafmethod) {
 398                                 for (p = 0; p < INT_TMP_CNT; p++)
 399                                         M_LST(rd->tmpintregs[p], REG_SP, (INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 400
 401                                 for (p = 0; p < FLT_TMP_CNT; p++)
 402                                         M_DST(rd->tmpfltregs[p], REG_SP, (INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 403                         }
 404
 405                         s1 += INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT;
 406                 }
 407
 408                 /* decide which monitor enter function to call */
 409
 410                 if (m->flags & ACC_STATIC) {
 411                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 412                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 413                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 414                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 415                         x86_64_call_reg(cd, REG_ITMP1);
 416
 417                 } else {
 418                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 419                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 420                         codegen_addxnullrefs(cd, cd->mcodeptr);
 421                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 422                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 423                         x86_64_call_reg(cd, REG_ITMP1);
 424                 }
 425
 426                 if (runverbose) {
 427                         for (p = 0; p < INT_ARG_CNT; p++)
 428                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 429
 430                         for (p = 0; p < FLT_ARG_CNT; p++)
 431                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 432
 433
 434                         if (m->isleafmethod) {
 435                                 for (p = 0; p < INT_TMP_CNT; p++)
 436                                         M_LLD(rd->tmpintregs[p], REG_SP, (INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 437
 438                                 for (p = 0; p < FLT_TMP_CNT; p++)
 439                                         M_DLD(rd->tmpfltregs[p], REG_SP, (INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 440                         }
 441
 442                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT) * 8, REG_SP);
 443                 }
 444         }
 445 #endif
 446
 447         /* Copy argument registers to stack and call trace function with pointer  */
 448         /* to arguments on stack.                                                 */
 449
 450         if (runverbose || opt_stat) {
 451                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 452
 453                 /* save integer argument registers */
 454
 455                 for (p = 0; p < INT_ARG_CNT; p++)
 456                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 457
 458                 /* save float argument registers */
 459
 460                 for (p = 0; p < FLT_ARG_CNT; p++)
 461                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 462
 463                 /* save temporary registers for leaf methods */
 464
 465                 if (m->isleafmethod) {
 466                         for (p = 0; p < INT_TMP_CNT; p++)
 467                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 468
 469                         for (p = 0; p < FLT_TMP_CNT; p++)
 470                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 471                 }
 472
 473                 if (runverbose) {
 474                         /* show integer hex code for float arguments */
 475
 476                         for (p = 0, l = 0; p < m->paramcount && p < INT_ARG_CNT; p++) {
 477                                 t = m->paramtypes[p];
 478
 479                                 /* if the paramtype is a float, we have to right shift all        */
 480                                 /* following integer registers                                    */
 481
 482                                 if (IS_FLT_DBL_TYPE(t)) {
 483                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 484                                                 x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
 485                                         }
 486
 487                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 488                                         l++;
 489                                 }
 490                         }
 491
 492                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 493                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 494                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 495                         x86_64_call_reg(cd, REG_ITMP1);
 496                 }
 497                 STATS({
 498                         x86_64_mov_imm_reg(cd,(u8)compiledinvokation,REG_ITMP1);
 499                         x86_64_call_reg(cd,REG_ITMP1);
 500                 })
 501
 502                 /* restore integer argument registers */
 503
 504                 for (p = 0; p < INT_ARG_CNT; p++)
 505                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 506
 507                 /* restore float argument registers */
 508
 509                 for (p = 0; p < FLT_ARG_CNT; p++)
 510                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 511
 512                 /* restore temporary registers for leaf methods */
 513
 514                 if (m->isleafmethod) {
 515                         for (p = 0; p < INT_TMP_CNT; p++)
 516                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 517
 518                         for (p = 0; p < FLT_TMP_CNT; p++)
 519                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 520                 }
 521
 522                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 523         }
 524
 525         }
 526
 527         /* end of header generation */
 528
 529         /* walk through all basic blocks */
 530         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 531
 532                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 533
 534                 if (bptr->flags >= BBREACHED) {
 535
 536                         /* branch resolving */
 537
 538                         branchref *bref;
 539                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 540                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 541                                                                   bref->branchpos,
 542                                                                   bptr->mpc);
 543                         }
 544
 545                 /* copy interface registers to their destination */
 546
 547                 src = bptr->instack;
 548                 len = bptr->indepth;
 549                 MCODECHECK(64 + len);
 550
 551 #ifdef LSRA
 552                 if (opt_lsra) {
 553                         while (src != NULL) {
 554                                 len--;
 555                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 556                                         if (bptr->type == BBTYPE_SBR) {
 557                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 558                                                 if (!(src->flags & INMEMORY))
 559                                                         d= src->regoff;
 560                                                 else
 561                                                         d=REG_ITMP1;
 562                                                 x86_64_pop_reg(cd, d);
 563                                                 store_reg_to_var_int(src, d);
 564
 565                                         } else if (bptr->type == BBTYPE_EXH) {
 566                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 567                                                 if (!(src->flags & INMEMORY))
 568                                                         d= src->regoff;
 569                                                 else
 570                                                         d=REG_ITMP1;
 571                                                 M_INTMOVE(REG_ITMP1, d);
 572                                                 store_reg_to_var_int(src, d);
 573                                         }
 574                                 }
 575                                 src = src->prev;
 576                         }
 577
 578                 } else {
 579 #endif
 580
 581         while (src != NULL) {
 582                         len--;
 583                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 584                                 if (bptr->type == BBTYPE_SBR) {
 585                                         d = reg_of_var(rd, src, REG_ITMP1);
 586                                         x86_64_pop_reg(cd, d);
 587                                         store_reg_to_var_int(src, d);
 588
 589                                 } else if (bptr->type == BBTYPE_EXH) {
 590                                         d = reg_of_var(rd, src, REG_ITMP1);
 591                                         M_INTMOVE(REG_ITMP1, d);
 592                                         store_reg_to_var_int(src, d);
 593                                 }
 594
 595                         } else {
 596                                 d = reg_of_var(rd, src, REG_ITMP1);
 597                                 if ((src->varkind != STACKVAR)) {
 598                                         s2 = src->type;
 599                                         if (IS_FLT_DBL_TYPE(s2)) {
 600                                                 s1 = rd->interfaces[len][s2].regoff;
 601                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 602                                                         M_FLTMOVE(s1, d);
 603
 604                                                 } else {
 605                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 606                                                 }
 607                                                 store_reg_to_var_flt(src, d);
 608
 609                                         } else {
 610                                                 s1 = rd->interfaces[len][s2].regoff;
 611                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 612                                                         M_INTMOVE(s1, d);
 613
 614                                                 } else {
 615                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 616                                                 }
 617                                                 store_reg_to_var_int(src, d);
 618                                         }
 619                                 }
 620                         }
 621                         src = src->prev;
 622                 }
 623 #ifdef LSRA
 624                 }
 625 #endif
 626                 /* walk through all instructions */
 627
 628                 src = bptr->instack;
 629                 len = bptr->icount;
 630                 currentline=0;
 631                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 632                         if (iptr->line != currentline) {
 633                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 634                                 /*printf("%s : %d\n",m->name->text,iptr->line);*/
 635                                 currentline = iptr->line;
 636                         }
 637
 638                         MCODECHECK(128);    /* XXX are 128 bytes enough? */
 639
 640                         switch (iptr->opc) {
 641                         case ICMD_INLINE_START: /* internal ICMDs                         */
 642                         case ICMD_INLINE_END:
 643                                 break;
 644
 645                         case ICMD_NOP:    /* ...  ==> ...                                 */
 646                                 break;
 647
 648                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 649                                 if (src->flags & INMEMORY) {
 650                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 651
 652                                 } else {
 653                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 654                                 }
 655                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 656                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 657                                 break;
 658
 659                 /* constant operations ************************************************/
 660
 661                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 662                                       /* op1 = 0, val.i = constant                    */
 663
 664                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 665                         if (iptr->val.i == 0) {
 666                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 667                         } else {
 668                                 x86_64_movl_imm_reg(cd, iptr->val.i, d);
 669                         }
 670                         store_reg_to_var_int(iptr->dst, d);
 671                         break;
 672
 673                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 674                                       /* op1 = 0, val.a = constant                    */
 675
 676                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 677                         if (iptr->val.a == 0) {
 678                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 679                         } else {
 680                                 x86_64_mov_imm_reg(cd, (s8) iptr->val.a, d);
 681                         }
 682                         store_reg_to_var_int(iptr->dst, d);
 683                         break;
 684
 685                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 686                                       /* op1 = 0, val.l = constant                    */
 687
 688                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 689                         if (iptr->val.l == 0) {
 690                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
 691                         } else {
 692                                 x86_64_mov_imm_reg(cd, iptr->val.l, d);
 693                         }
 694                         store_reg_to_var_int(iptr->dst, d);
 695                         break;
 696
 697                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 698                                       /* op1 = 0, val.f = constant                    */
 699
 700                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 701                         a = dseg_addfloat(cd, iptr->val.f);
 702                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + a, d);
 703                         store_reg_to_var_flt(iptr->dst, d);
 704                         break;
 705
 706                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 707                                       /* op1 = 0, val.d = constant                    */
 708
 709                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 710                         a = dseg_adddouble(cd, iptr->val.d);
 711                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, d);
 712                         store_reg_to_var_flt(iptr->dst, d);
 713                         break;
 714
 715
 716                 /* load/store operations **********************************************/
 717
 718                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 719                                       /* op1 = local variable                         */
 720
 721                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 722                         if ((iptr->dst->varkind == LOCALVAR) &&
 723                             (iptr->dst->varnum == iptr->op1)) {
 724                                 break;
 725                         }
 726                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 727                         if (var->flags & INMEMORY) {
 728                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 729                                 store_reg_to_var_int(iptr->dst, d);
 730
 731                         } else {
 732                                 if (iptr->dst->flags & INMEMORY) {
 733                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 734
 735                                 } else {
 736                                         M_INTMOVE(var->regoff, d);
 737                                 }
 738                         }
 739                         break;
 740
 741                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 742                 case ICMD_ALOAD:      /* op1 = local variable                         */
 743
 744                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 745                         if ((iptr->dst->varkind == LOCALVAR) &&
 746                             (iptr->dst->varnum == iptr->op1)) {
 747                                 break;
 748                         }
 749                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 750                         if (var->flags & INMEMORY) {
 751                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 752                                 store_reg_to_var_int(iptr->dst, d);
 753
 754                         } else {
 755                                 if (iptr->dst->flags & INMEMORY) {
 756                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 757
 758                                 } else {
 759                                         M_INTMOVE(var->regoff, d);
 760                                 }
 761                         }
 762                         break;
 763
 764                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 765                 case ICMD_DLOAD:      /* op1 = local variable                         */
 766
 767                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 768                         if ((iptr->dst->varkind == LOCALVAR) &&
 769                             (iptr->dst->varnum == iptr->op1)) {
 770                                 break;
 771                         }
 772                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 773                         if (var->flags & INMEMORY) {
 774                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 775                                 store_reg_to_var_flt(iptr->dst, d);
 776
 777                         } else {
 778                                 if (iptr->dst->flags & INMEMORY) {
 779                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 780
 781                                 } else {
 782                                         M_FLTMOVE(var->regoff, d);
 783                                 }
 784                         }
 785                         break;
 786
 787                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 788                 case ICMD_LSTORE:     /* op1 = local variable                         */
 789                 case ICMD_ASTORE:
 790
 791                         if ((src->varkind == LOCALVAR) &&
 792                             (src->varnum == iptr->op1)) {
 793                                 break;
 794                         }
 795                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 796                         if (var->flags & INMEMORY) {
 797                                 var_to_reg_int(s1, src, REG_ITMP1);
 798                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 799
 800                         } else {
 801                                 var_to_reg_int(s1, src, var->regoff);
 802                                 M_INTMOVE(s1, var->regoff);
 803                         }
 804                         break;
 805
 806                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 807                 case ICMD_DSTORE:     /* op1 = local variable                         */
 808
 809                         if ((src->varkind == LOCALVAR) &&
 810                             (src->varnum == iptr->op1)) {
 811                                 break;
 812                         }
 813                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 814                         if (var->flags & INMEMORY) {
 815                                 var_to_reg_flt(s1, src, REG_FTMP1);
 816                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 817
 818                         } else {
 819                                 var_to_reg_flt(s1, src, var->regoff);
 820                                 M_FLTMOVE(s1, var->regoff);
 821                         }
 822                         break;
 823
 824
 825                 /* pop/dup/swap operations ********************************************/
 826
 827                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 828
 829                 case ICMD_POP:        /* ..., value  ==> ...                          */
 830                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 831                         break;
 832
 833                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 834                         M_COPY(src, iptr->dst);
 835                         break;
 836
 837                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 838
 839                         M_COPY(src,       iptr->dst);
 840                         M_COPY(src->prev, iptr->dst->prev);
 841                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 842                         break;
 843
 844                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 845
 846                         M_COPY(src,             iptr->dst);
 847                         M_COPY(src->prev,       iptr->dst->prev);
 848                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 849                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 850                         break;
 851
 852                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 853
 854                         M_COPY(src,       iptr->dst);
 855                         M_COPY(src->prev, iptr->dst->prev);
 856                         break;
 857
 858                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 859
 860                         M_COPY(src,             iptr->dst);
 861                         M_COPY(src->prev,       iptr->dst->prev);
 862                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 863                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 864                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 865                         break;
 866
 867                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 868
 869                         M_COPY(src,                   iptr->dst);
 870                         M_COPY(src->prev,             iptr->dst->prev);
 871                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 872                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 873                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 874                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 875                         break;
 876
 877                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 878
 879                         M_COPY(src,       iptr->dst->prev);
 880                         M_COPY(src->prev, iptr->dst);
 881                         break;
 882
 883
 884                 /* integer operations *************************************************/
 885
 886                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 887
 888                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 889                         if (iptr->dst->flags & INMEMORY) {
 890                                 if (src->flags & INMEMORY) {
 891                                         if (src->regoff == iptr->dst->regoff) {
 892                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 893
 894                                         } else {
 895                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 896                                                 x86_64_negl_reg(cd, REG_ITMP1);
 897                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 898                                         }
 899
 900                                 } else {
 901                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 902                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 903                                 }
 904
 905                         } else {
 906                                 if (src->flags & INMEMORY) {
 907                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 908                                         x86_64_negl_reg(cd, d);
 909
 910                                 } else {
 911                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 912                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 913                                 }
 914                         }
 915                         break;
 916
 917                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 918
 919                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 920                         if (iptr->dst->flags & INMEMORY) {
 921                                 if (src->flags & INMEMORY) {
 922                                         if (src->regoff == iptr->dst->regoff) {
 923                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 924
 925                                         } else {
 926                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 927                                                 x86_64_neg_reg(cd, REG_ITMP1);
 928                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 929                                         }
 930
 931                                 } else {
 932                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 933                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 934                                 }
 935
 936                         } else {
 937                                 if (src->flags & INMEMORY) {
 938                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 939                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 940
 941                                 } else {
 942                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 943                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 944                                 }
 945                         }
 946                         break;
 947
 948                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 949
 950                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 951                         if (src->flags & INMEMORY) {
 952                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 953
 954                         } else {
 955                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 956                         }
 957                         store_reg_to_var_int(iptr->dst, d);
 958                         break;
 959
 960                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 961
 962                         var_to_reg_int(s1, src, REG_ITMP1);
 963                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 964                         M_INTMOVE(s1, d);
 965                         store_reg_to_var_int(iptr->dst, d);
 966                         break;
 967
 968                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 969
 970                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 971                         if (src->flags & INMEMORY) {
 972                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 973
 974                         } else {
 975                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 976                         }
 977                         store_reg_to_var_int(iptr->dst, d);
 978                         break;
 979
 980                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 981
 982                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 983                         if (src->flags & INMEMORY) {
 984                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 985
 986                         } else {
 987                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 988                         }
 989                         store_reg_to_var_int(iptr->dst, d);
 990                         break;
 991
 992                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 993
 994                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 995                         if (src->flags & INMEMORY) {
 996                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 997
 998                         } else {
 999                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
1000                         }
1001                         store_reg_to_var_int(iptr->dst, d);
1002                         break;
1003
1004
1005                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1006
1007                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1008                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
1009                         break;
1010
1011                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
1012                                       /* val.i = constant                             */
1013
1014                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1015                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
1016                         break;
1017
1018                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1019
1020                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1021                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
1022                         break;
1023
1024                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
1025                                       /* val.l = constant                             */
1026
1027                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1028                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
1029                         break;
1030
1031                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1032
1033                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1034                         if (iptr->dst->flags & INMEMORY) {
1035                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1036                                         if (src->prev->regoff == iptr->dst->regoff) {
1037                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1038                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1039
1040                                         } else {
1041                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1042                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1043                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1044                                         }
1045
1046                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1047                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1048                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1049                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1050
1051                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1052                                         if (src->prev->regoff == iptr->dst->regoff) {
1053                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1054
1055                                         } else {
1056                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1057                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1058                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1059                                         }
1060
1061                                 } else {
1062                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1063                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1064                                 }
1065
1066                         } else {
1067                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1068                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1069                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1070
1071                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1072                                         M_INTMOVE(src->prev->regoff, d);
1073                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1074
1075                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1076                                         /* workaround for reg alloc */
1077                                         if (src->regoff == iptr->dst->regoff) {
1078                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1079                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1080                                                 M_INTMOVE(REG_ITMP1, d);
1081
1082                                         } else {
1083                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1084                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
1085                                         }
1086
1087                                 } else {
1088                                         /* workaround for reg alloc */
1089                                         if (src->regoff == iptr->dst->regoff) {
1090                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1091                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1092                                                 M_INTMOVE(REG_ITMP1, d);
1093
1094                                         } else {
1095                                                 M_INTMOVE(src->prev->regoff, d);
1096                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
1097                                         }
1098                                 }
1099                         }
1100                         break;
1101
1102                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
1103                                       /* val.i = constant                             */
1104
1105                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1106                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
1107                         break;
1108
1109                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1110
1111                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1112                         if (iptr->dst->flags & INMEMORY) {
1113                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1114                                         if (src->prev->regoff == iptr->dst->regoff) {
1115                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1116                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1117
1118                                         } else {
1119                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1120                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1121                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1122                                         }
1123
1124                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1125                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1126                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1127                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1128
1129                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1130                                         if (src->prev->regoff == iptr->dst->regoff) {
1131                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1132
1133                                         } else {
1134                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1135                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1136                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1137                                         }
1138
1139                                 } else {
1140                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1141                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1142                                 }
1143
1144                         } else {
1145                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1146                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1147                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1148
1149                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1150                                         M_INTMOVE(src->prev->regoff, d);
1151                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1152
1153                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1154                                         /* workaround for reg alloc */
1155                                         if (src->regoff == iptr->dst->regoff) {
1156                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1157                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1158                                                 M_INTMOVE(REG_ITMP1, d);
1159
1160                                         } else {
1161                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1162                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1163                                         }
1164
1165                                 } else {
1166                                         /* workaround for reg alloc */
1167                                         if (src->regoff == iptr->dst->regoff) {
1168                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1169                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1170                                                 M_INTMOVE(REG_ITMP1, d);
1171
1172                                         } else {
1173                                                 M_INTMOVE(src->prev->regoff, d);
1174                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1175                                         }
1176                                 }
1177                         }
1178                         break;
1179
1180                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1181                                       /* val.l = constant                             */
1182
1183                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1184                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1185                         break;
1186
1187                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1188
1189                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1190                         if (iptr->dst->flags & INMEMORY) {
1191                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1192                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1193                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1194                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1195
1196                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1197                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1198                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1199                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1200
1201                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1202                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1203                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1204                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1205
1206                                 } else {
1207                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1208                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1209                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1210                                 }
1211
1212                         } else {
1213                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1214                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1215                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1216
1217                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1218                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1219                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1220
1221                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1222                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1223                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1224
1225                                 } else {
1226                                         if (src->regoff == iptr->dst->regoff) {
1227                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1228
1229                                         } else {
1230                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1231                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1232                                         }
1233                                 }
1234                         }
1235                         break;
1236
1237                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1238                                       /* val.i = constant                             */
1239
1240                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1241                         if (iptr->dst->flags & INMEMORY) {
1242                                 if (src->flags & INMEMORY) {
1243                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1244                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1245
1246                                 } else {
1247                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1248                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1249                                 }
1250
1251                         } else {
1252                                 if (src->flags & INMEMORY) {
1253                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1254
1255                                 } else {
1256                                         if (iptr->val.i == 2) {
1257                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1258                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1259
1260                                         } else {
1261                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1262                                         }
1263                                 }
1264                         }
1265                         break;
1266
1267                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1268
1269                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1270                         if (iptr->dst->flags & INMEMORY) {
1271                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1272                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1273                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1274                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1275
1276                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1277                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1278                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1279                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1280
1281                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1282                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1283                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1284                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1285
1286                                 } else {
1287                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1288                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1289                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1290                                 }
1291
1292                         } else {
1293                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1294                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1295                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1296
1297                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1298                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1299                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1300
1301                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1302                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1303                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1304
1305                                 } else {
1306                                         if (src->regoff == iptr->dst->regoff) {
1307                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1308
1309                                         } else {
1310                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1311                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1312                                         }
1313                                 }
1314                         }
1315                         break;
1316
1317                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1318                                       /* val.l = constant                             */
1319
1320                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1321                         if (iptr->dst->flags & INMEMORY) {
1322                                 if (src->flags & INMEMORY) {
1323                                         if (IS_IMM32(iptr->val.l)) {
1324                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1325
1326                                         } else {
1327                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1328                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1329                                         }
1330                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1331
1332                                 } else {
1333                                         if (IS_IMM32(iptr->val.l)) {
1334                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1335
1336                                         } else {
1337                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1338                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1339                                         }
1340                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1341                                 }
1342
1343                         } else {
1344                                 if (src->flags & INMEMORY) {
1345                                         if (IS_IMM32(iptr->val.l)) {
1346                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1347
1348                                         } else {
1349                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1350                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1351                                         }
1352
1353                                 } else {
1354                                         /* should match in many cases */
1355                                         if (iptr->val.l == 2) {
1356                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1357                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1358
1359                                         } else {
1360                                                 if (IS_IMM32(iptr->val.l)) {
1361                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1362
1363                                                 } else {
1364                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1365                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1366                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1367                                                 }
1368                                         }
1369                                 }
1370                         }
1371                         break;
1372
1373                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1374
1375                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1376                 if (src->prev->flags & INMEMORY) {
1377                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1378
1379                         } else {
1380                                 M_INTMOVE(src->prev->regoff, RAX);
1381                         }
1382
1383                         if (src->flags & INMEMORY) {
1384                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1385
1386                         } else {
1387                                 M_INTMOVE(src->regoff, REG_ITMP3);
1388                         }
1389                         gen_div_check(src);
1390
1391                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1392                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1393                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1394                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                  /* 6 bytes */
1395
1396                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1397                         x86_64_cltd(cd);
1398                         x86_64_idivl_reg(cd, REG_ITMP3);
1399
1400                         if (iptr->dst->flags & INMEMORY) {
1401                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1402                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1403
1404                         } else {
1405                                 M_INTMOVE(RAX, iptr->dst->regoff);
1406
1407                                 if (iptr->dst->regoff != RDX) {
1408                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1409                                 }
1410                         }
1411                         break;
1412
1413                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1414                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1415                         if (src->prev->flags & INMEMORY) {
1416                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1417
1418                         } else {
1419                                 M_INTMOVE(src->prev->regoff, RAX);
1420                         }
1421
1422                         if (src->flags & INMEMORY) {
1423                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1424
1425                         } else {
1426                                 M_INTMOVE(src->regoff, REG_ITMP3);
1427                         }
1428                         gen_div_check(src);
1429
1430                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1431
1432                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX);    /* check as described in jvm spec */
1433                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1434
1435
1436                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);           /* 2 bytes */
1437                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);      /* 4 bytes */
1438                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                      /* 6 bytes */
1439
1440                         x86_64_cltd(cd);
1441                         x86_64_idivl_reg(cd, REG_ITMP3);
1442
1443                         if (iptr->dst->flags & INMEMORY) {
1444                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1445                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1446
1447                         } else {
1448                                 M_INTMOVE(RDX, iptr->dst->regoff);
1449
1450                                 if (iptr->dst->regoff != RDX) {
1451                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1452                                 }
1453                         }
1454                         break;
1455
1456                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1457                                       /* val.i = constant                             */
1458
1459                         var_to_reg_int(s1, src, REG_ITMP1);
1460                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1461                         M_INTMOVE(s1, REG_ITMP1);
1462                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1463                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1464                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1465                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1466                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1467                         store_reg_to_var_int(iptr->dst, d);
1468                         break;
1469
1470                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1471                                       /* val.i = constant                             */
1472
1473                         var_to_reg_int(s1, src, REG_ITMP1);
1474                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1475                         M_INTMOVE(s1, REG_ITMP1);
1476                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1477                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1478                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1479                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1480                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1481                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1482                         store_reg_to_var_int(iptr->dst, d);
1483                         break;
1484
1485
1486                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1487
1488                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1489                 if (src->prev->flags & INMEMORY) {
1490                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1491
1492                         } else {
1493                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1494                         }
1495
1496                         if (src->flags & INMEMORY) {
1497                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1498
1499                         } else {
1500                                 M_INTMOVE(src->regoff, REG_ITMP3);
1501                         }
1502                         gen_div_check(src);
1503
1504                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1505                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1506                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1507                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1508                         x86_64_jcc(cd, X86_64_CC_E, 3 + 2 + 3);                     /* 6 bytes */
1509
1510                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1511                         x86_64_cqto(cd);
1512                         x86_64_idiv_reg(cd, REG_ITMP3);
1513
1514                         if (iptr->dst->flags & INMEMORY) {
1515                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1516                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1517
1518                         } else {
1519                                 M_INTMOVE(RAX, iptr->dst->regoff);
1520
1521                                 if (iptr->dst->regoff != RDX) {
1522                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1523                                 }
1524                         }
1525                         break;
1526
1527                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1528
1529                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1530                         if (src->prev->flags & INMEMORY) {
1531                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1532
1533                         } else {
1534                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1535                         }
1536
1537                         if (src->flags & INMEMORY) {
1538                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1539
1540                         } else {
1541                                 M_INTMOVE(src->regoff, REG_ITMP3);
1542                         }
1543                         gen_div_check(src);
1544
1545                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2);    /* save %rdx, cause it's an argument register */
1546
1547                         x86_64_mov_imm_reg(cd, 0x8000000000000000LL, REG_ITMP2);    /* check as described in jvm spec */
1548                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, REG_ITMP1);
1549                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1550
1551
1552                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);              /* 2 bytes */
1553                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);          /* 4 bytes */
1554                         x86_64_jcc(cd, X86_64_CC_E, 2 + 3);                         /* 6 bytes */
1555
1556                         x86_64_cqto(cd);
1557                         x86_64_idiv_reg(cd, REG_ITMP3);
1558
1559                         if (iptr->dst->flags & INMEMORY) {
1560                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1561                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1562
1563                         } else {
1564                                 M_INTMOVE(RDX, iptr->dst->regoff);
1565
1566                                 if (iptr->dst->regoff != RDX) {
1567                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);    /* restore %rdx */
1568                                 }
1569                         }
1570                         break;
1571
1572                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1573                                       /* val.i = constant                             */
1574
1575                         var_to_reg_int(s1, src, REG_ITMP1);
1576                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1577                         M_INTMOVE(s1, REG_ITMP1);
1578                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1579                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1580                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1581                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1582                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1583                         store_reg_to_var_int(iptr->dst, d);
1584                         break;
1585
1586                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1587                                       /* val.l = constant                             */
1588
1589                         var_to_reg_int(s1, src, REG_ITMP1);
1590                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1591                         M_INTMOVE(s1, REG_ITMP1);
1592                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1593                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1594                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1595                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1596                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1597                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1598                         store_reg_to_var_int(iptr->dst, d);
1599                         break;
1600
1601                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1602
1603                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1604                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1605                         break;
1606
1607                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1608                                       /* val.i = constant                             */
1609
1610                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1611                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1612                         break;
1613
1614                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1615
1616                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1617                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1618                         break;
1619
1620                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1621                                       /* val.i = constant                             */
1622
1623                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1624                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1625                         break;
1626
1627                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1628
1629                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1630                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1631                         break;
1632
1633                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1634                                       /* val.i = constant                             */
1635
1636                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1637                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1638                         break;
1639
1640                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1641
1642                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1643                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1644                         break;
1645
1646         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1647                                           /* val.i = constant                             */
1648
1649                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1650                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1651                         break;
1652
1653                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1654
1655                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1656                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1657                         break;
1658
1659                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1660                                       /* val.i = constant                             */
1661
1662                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1663                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1664                         break;
1665
1666                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1667
1668                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1669                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1670                         break;
1671
1672                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1673                                       /* val.l = constant                             */
1674
1675                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1676                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1677                         break;
1678
1679                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1680
1681                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1682                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1683                         break;
1684
1685                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1686                                       /* val.i = constant                             */
1687
1688                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1689                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1690                         break;
1691
1692                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1693
1694                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1695                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1696                         break;
1697
1698                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1699                                       /* val.l = constant                             */
1700
1701                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1702                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1703                         break;
1704
1705                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1706
1707                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1708                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1709                         break;
1710
1711                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1712                                       /* val.i = constant                             */
1713
1714                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1715                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1716                         break;
1717
1718                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1719
1720                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1721                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1722                         break;
1723
1724                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1725                                       /* val.l = constant                             */
1726
1727                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1728                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1729                         break;
1730
1731                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1732
1733                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1734                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1735                         break;
1736
1737                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1738                                       /* val.i = constant                             */
1739
1740                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1741                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1742                         break;
1743
1744                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1745
1746                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1747                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1748                         break;
1749
1750                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1751                                       /* val.l = constant                             */
1752
1753                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1754                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1755                         break;
1756
1757
1758                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1759                                       /* op1 = variable, val.i = constant             */
1760
1761                         /* using inc and dec is definitely faster than add -- tested      */
1762                         /* with sieve                                                     */
1763
1764                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1765                         d = var->regoff;
1766                         if (var->flags & INMEMORY) {
1767                                 if (iptr->val.i == 1) {
1768                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1769
1770                                 } else if (iptr->val.i == -1) {
1771                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1772
1773                                 } else {
1774                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1775                                 }
1776
1777                         } else {
1778                                 if (iptr->val.i == 1) {
1779                                         x86_64_incl_reg(cd, d);
1780
1781                                 } else if (iptr->val.i == -1) {
1782                                         x86_64_decl_reg(cd, d);
1783
1784                                 } else {
1785                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1786                                 }
1787                         }
1788                         break;
1789
1790
1791                 /* floating operations ************************************************/
1792
1793                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1794
1795                         var_to_reg_flt(s1, src, REG_FTMP1);
1796                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1797                         a = dseg_adds4(cd, 0x80000000);
1798                         M_FLTMOVE(s1, d);
1799                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1800                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1801                         store_reg_to_var_flt(iptr->dst, d);
1802                         break;
1803
1804                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1805
1806                         var_to_reg_flt(s1, src, REG_FTMP1);
1807                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1808                         a = dseg_adds8(cd, 0x8000000000000000);
1809                         M_FLTMOVE(s1, d);
1810                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + a, REG_FTMP2);
1811                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1812                         store_reg_to_var_flt(iptr->dst, d);
1813                         break;
1814
1815                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1816
1817                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1818                         var_to_reg_flt(s2, src, REG_FTMP2);
1819                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1820                         if (s1 == d) {
1821                                 x86_64_addss_reg_reg(cd, s2, d);
1822                         } else if (s2 == d) {
1823                                 x86_64_addss_reg_reg(cd, s1, d);
1824                         } else {
1825                                 M_FLTMOVE(s1, d);
1826                                 x86_64_addss_reg_reg(cd, s2, d);
1827                         }
1828                         store_reg_to_var_flt(iptr->dst, d);
1829                         break;
1830
1831                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1832
1833                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1834                         var_to_reg_flt(s2, src, REG_FTMP2);
1835                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1836                         if (s1 == d) {
1837                                 x86_64_addsd_reg_reg(cd, s2, d);
1838                         } else if (s2 == d) {
1839                                 x86_64_addsd_reg_reg(cd, s1, d);
1840                         } else {
1841                                 M_FLTMOVE(s1, d);
1842                                 x86_64_addsd_reg_reg(cd, s2, d);
1843                         }
1844                         store_reg_to_var_flt(iptr->dst, d);
1845                         break;
1846
1847                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1848
1849                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1850                         var_to_reg_flt(s2, src, REG_FTMP2);
1851                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1852                         if (s2 == d) {
1853                                 M_FLTMOVE(s2, REG_FTMP2);
1854                                 s2 = REG_FTMP2;
1855                         }
1856                         M_FLTMOVE(s1, d);
1857                         x86_64_subss_reg_reg(cd, s2, d);
1858                         store_reg_to_var_flt(iptr->dst, d);
1859                         break;
1860
1861                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1862
1863                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1864                         var_to_reg_flt(s2, src, REG_FTMP2);
1865                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1866                         if (s2 == d) {
1867                                 M_FLTMOVE(s2, REG_FTMP2);
1868                                 s2 = REG_FTMP2;
1869                         }
1870                         M_FLTMOVE(s1, d);
1871                         x86_64_subsd_reg_reg(cd, s2, d);
1872                         store_reg_to_var_flt(iptr->dst, d);
1873                         break;
1874
1875                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1876
1877                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1878                         var_to_reg_flt(s2, src, REG_FTMP2);
1879                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1880                         if (s1 == d) {
1881                                 x86_64_mulss_reg_reg(cd, s2, d);
1882                         } else if (s2 == d) {
1883                                 x86_64_mulss_reg_reg(cd, s1, d);
1884                         } else {
1885                                 M_FLTMOVE(s1, d);
1886                                 x86_64_mulss_reg_reg(cd, s2, d);
1887                         }
1888                         store_reg_to_var_flt(iptr->dst, d);
1889                         break;
1890
1891                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1892
1893                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1894                         var_to_reg_flt(s2, src, REG_FTMP2);
1895                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1896                         if (s1 == d) {
1897                                 x86_64_mulsd_reg_reg(cd, s2, d);
1898                         } else if (s2 == d) {
1899                                 x86_64_mulsd_reg_reg(cd, s1, d);
1900                         } else {
1901                                 M_FLTMOVE(s1, d);
1902                                 x86_64_mulsd_reg_reg(cd, s2, d);
1903                         }
1904                         store_reg_to_var_flt(iptr->dst, d);
1905                         break;
1906
1907                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1908
1909                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1910                         var_to_reg_flt(s2, src, REG_FTMP2);
1911                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1912                         if (s2 == d) {
1913                                 M_FLTMOVE(s2, REG_FTMP2);
1914                                 s2 = REG_FTMP2;
1915                         }
1916                         M_FLTMOVE(s1, d);
1917                         x86_64_divss_reg_reg(cd, s2, d);
1918                         store_reg_to_var_flt(iptr->dst, d);
1919                         break;
1920
1921                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1922
1923                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1924                         var_to_reg_flt(s2, src, REG_FTMP2);
1925                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1926                         if (s2 == d) {
1927                                 M_FLTMOVE(s2, REG_FTMP2);
1928                                 s2 = REG_FTMP2;
1929                         }
1930                         M_FLTMOVE(s1, d);
1931                         x86_64_divsd_reg_reg(cd, s2, d);
1932                         store_reg_to_var_flt(iptr->dst, d);
1933                         break;
1934
1935                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1936
1937                         var_to_reg_int(s1, src, REG_ITMP1);
1938                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1939                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1940                         store_reg_to_var_flt(iptr->dst, d);
1941                         break;
1942
1943                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1944
1945                         var_to_reg_int(s1, src, REG_ITMP1);
1946                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1947                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1948                         store_reg_to_var_flt(iptr->dst, d);
1949                         break;
1950
1951                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1952
1953                         var_to_reg_int(s1, src, REG_ITMP1);
1954                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1955                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1956                         store_reg_to_var_flt(iptr->dst, d);
1957                         break;
1958
1959                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1960
1961                         var_to_reg_int(s1, src, REG_ITMP1);
1962                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1963                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1964                         store_reg_to_var_flt(iptr->dst, d);
1965                         break;
1966
1967                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1968
1969                         var_to_reg_flt(s1, src, REG_FTMP1);
1970                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1971                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1972                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1973                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1974                         x86_64_jcc(cd, X86_64_CC_NE, a);
1975                         M_FLTMOVE(s1, REG_FTMP1);
1976                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1977                         x86_64_call_reg(cd, REG_ITMP2);
1978                         M_INTMOVE(REG_RESULT, d);
1979                         store_reg_to_var_int(iptr->dst, d);
1980                         break;
1981
1982                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1983
1984                         var_to_reg_flt(s1, src, REG_FTMP1);
1985                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1986                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1987                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1988                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1989                         x86_64_jcc(cd, X86_64_CC_NE, a);
1990                         M_FLTMOVE(s1, REG_FTMP1);
1991                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1992                         x86_64_call_reg(cd, REG_ITMP2);
1993                         M_INTMOVE(REG_RESULT, d);
1994                         store_reg_to_var_int(iptr->dst, d);
1995                         break;
1996
1997                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1998
1999                         var_to_reg_flt(s1, src, REG_FTMP1);
2000                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2001                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
2002                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
2003                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
2004                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
2005                         x86_64_jcc(cd, X86_64_CC_NE, a);
2006                         M_FLTMOVE(s1, REG_FTMP1);
2007                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
2008                         x86_64_call_reg(cd, REG_ITMP2);
2009                         M_INTMOVE(REG_RESULT, d);
2010                         store_reg_to_var_int(iptr->dst, d);
2011                         break;
2012
2013                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
2014
2015                         var_to_reg_flt(s1, src, REG_FTMP1);
2016                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2017                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
2018                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
2019                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
2020                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
2021                         x86_64_jcc(cd, X86_64_CC_NE, a);
2022                         M_FLTMOVE(s1, REG_FTMP1);
2023                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
2024                         x86_64_call_reg(cd, REG_ITMP2);
2025                         M_INTMOVE(REG_RESULT, d);
2026                         store_reg_to_var_int(iptr->dst, d);
2027                         break;
2028
2029                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
2030
2031                         var_to_reg_flt(s1, src, REG_FTMP1);
2032                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2033                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
2034                         store_reg_to_var_flt(iptr->dst, d);
2035                         break;
2036
2037                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
2038
2039                         var_to_reg_flt(s1, src, REG_FTMP1);
2040                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2041                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
2042                         store_reg_to_var_flt(iptr->dst, d);
2043                         break;
2044
2045                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
2046                                           /* == => 0, < => 1, > => -1 */
2047
2048                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
2049                         var_to_reg_flt(s2, src, REG_FTMP2);
2050                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2051                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
2052                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
2053                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
2054                         x86_64_ucomiss_reg_reg(cd, s1, s2);
2055                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
2056                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
2057                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
2058                         store_reg_to_var_int(iptr->dst, d);
2059                         break;
2060
2061                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
2062                                           /* == => 0, < => 1, > => -1 */
2063
2064                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
2065                         var_to_reg_flt(s2, src, REG_FTMP2);
2066                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2067                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
2068                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
2069                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
2070                         x86_64_ucomiss_reg_reg(cd, s1, s2);
2071                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
2072                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
2073                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
2074                         store_reg_to_var_int(iptr->dst, d);
2075                         break;
2076
2077                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
2078                                           /* == => 0, < => 1, > => -1 */
2079
2080                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
2081                         var_to_reg_flt(s2, src, REG_FTMP2);
2082                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2083                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
2084                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
2085                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
2086                         x86_64_ucomisd_reg_reg(cd, s1, s2);
2087                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
2088                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
2089                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
2090                         store_reg_to_var_int(iptr->dst, d);
2091                         break;
2092
2093                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
2094                                           /* == => 0, < => 1, > => -1 */
2095
2096                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
2097                         var_to_reg_flt(s2, src, REG_FTMP2);
2098                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2099                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
2100                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
2101                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
2102                         x86_64_ucomisd_reg_reg(cd, s1, s2);
2103                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
2104                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
2105                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
2106                         store_reg_to_var_int(iptr->dst, d);
2107                         break;
2108
2109
2110                 /* memory operations **************************************************/
2111
2112                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
2113
2114                         var_to_reg_int(s1, src, REG_ITMP1);
2115                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2116                         gen_nullptr_check(s1);
2117                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
2118                         store_reg_to_var_int(iptr->dst, d);
2119                         break;
2120
2121                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2122
2123                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2124                         var_to_reg_int(s2, src, REG_ITMP2);
2125                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2126                         if (iptr->op1 == 0) {
2127                                 gen_nullptr_check(s1);
2128                                 gen_bound_check;
2129                         }
2130                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2131                         store_reg_to_var_int(iptr->dst, d);
2132                         break;
2133
2134                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2135
2136                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2137                         var_to_reg_int(s2, src, REG_ITMP2);
2138                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2139                         if (iptr->op1 == 0) {
2140                                 gen_nullptr_check(s1);
2141                                 gen_bound_check;
2142                         }
2143                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2144                         store_reg_to_var_int(iptr->dst, d);
2145                         break;
2146
2147                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2148
2149                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2150                         var_to_reg_int(s2, src, REG_ITMP2);
2151                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2152                         if (iptr->op1 == 0) {
2153                                 gen_nullptr_check(s1);
2154                                 gen_bound_check;
2155                         }
2156                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2157                         store_reg_to_var_int(iptr->dst, d);
2158                         break;
2159
2160                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2161
2162                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2163                         var_to_reg_int(s2, src, REG_ITMP2);
2164                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2165                         if (iptr->op1 == 0) {
2166                                 gen_nullptr_check(s1);
2167                                 gen_bound_check;
2168                         }
2169                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2170                         store_reg_to_var_flt(iptr->dst, d);
2171                         break;
2172
2173                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2174
2175                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2176                         var_to_reg_int(s2, src, REG_ITMP2);
2177                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2178                         if (iptr->op1 == 0) {
2179                                 gen_nullptr_check(s1);
2180                                 gen_bound_check;
2181                         }
2182                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2183                         store_reg_to_var_flt(iptr->dst, d);
2184                         break;
2185
2186                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2187
2188                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2189                         var_to_reg_int(s2, src, REG_ITMP2);
2190                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2191                         if (iptr->op1 == 0) {
2192                                 gen_nullptr_check(s1);
2193                                 gen_bound_check;
2194                         }
2195                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2196                         store_reg_to_var_int(iptr->dst, d);
2197                         break;
2198
2199                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2200
2201                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2202                         var_to_reg_int(s2, src, REG_ITMP2);
2203                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2204                         if (iptr->op1 == 0) {
2205                                 gen_nullptr_check(s1);
2206                                 gen_bound_check;
2207                         }
2208                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2209                         store_reg_to_var_int(iptr->dst, d);
2210                         break;
2211
2212                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2213
2214                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2215                         var_to_reg_int(s2, src, REG_ITMP2);
2216                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2217                         if (iptr->op1 == 0) {
2218                                 gen_nullptr_check(s1);
2219                                 gen_bound_check;
2220                         }
2221                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2222                         store_reg_to_var_int(iptr->dst, d);
2223                         break;
2224
2225
2226                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2227
2228                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2229                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2230                         if (iptr->op1 == 0) {
2231                                 gen_nullptr_check(s1);
2232                                 gen_bound_check;
2233                         }
2234                         var_to_reg_int(s3, src, REG_ITMP3);
2235                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2236                         break;
2237
2238                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2239
2240                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2241                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2242                         if (iptr->op1 == 0) {
2243                                 gen_nullptr_check(s1);
2244                                 gen_bound_check;
2245                         }
2246                         var_to_reg_int(s3, src, REG_ITMP3);
2247                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2248                         break;
2249
2250                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2251
2252                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2253                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2254                         if (iptr->op1 == 0) {
2255                                 gen_nullptr_check(s1);
2256                                 gen_bound_check;
2257                         }
2258                         var_to_reg_int(s3, src, REG_ITMP3);
2259                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2260                         break;
2261
2262                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2263
2264                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2265                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2266                         if (iptr->op1 == 0) {
2267                                 gen_nullptr_check(s1);
2268                                 gen_bound_check;
2269                         }
2270                         var_to_reg_flt(s3, src, REG_FTMP3);
2271                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2272                         break;
2273
2274                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2275
2276                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2277                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2278                         if (iptr->op1 == 0) {
2279                                 gen_nullptr_check(s1);
2280                                 gen_bound_check;
2281                         }
2282                         var_to_reg_flt(s3, src, REG_FTMP3);
2283                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2284                         break;
2285
2286                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2287
2288                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2289                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2290                         if (iptr->op1 == 0) {
2291                                 gen_nullptr_check(s1);
2292                                 gen_bound_check;
2293                         }
2294                         var_to_reg_int(s3, src, REG_ITMP3);
2295                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2296                         break;
2297
2298                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2299
2300                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2301                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2302                         if (iptr->op1 == 0) {
2303                                 gen_nullptr_check(s1);
2304                                 gen_bound_check;
2305                         }
2306                         var_to_reg_int(s3, src, REG_ITMP3);
2307                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2308                         break;
2309
2310                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2311
2312                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2313                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2314                         if (iptr->op1 == 0) {
2315                                 gen_nullptr_check(s1);
2316                                 gen_bound_check;
2317                         }
2318                         var_to_reg_int(s3, src, REG_ITMP3);
2319                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2320                         break;
2321
2322                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2323
2324                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2325                         var_to_reg_int(s2, src, REG_ITMP2);
2326                         if (iptr->op1 == 0) {
2327                                 gen_nullptr_check(s1);
2328                                 gen_bound_check;
2329                         }
2330                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2331                         break;
2332
2333                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2334
2335                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2336                         var_to_reg_int(s2, src, REG_ITMP2);
2337                         if (iptr->op1 == 0) {
2338                                 gen_nullptr_check(s1);
2339                                 gen_bound_check;
2340                         }
2341
2342                         if (IS_IMM32(iptr->val.l)) {
2343                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2344
2345                         } else {
2346                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2347                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2348                         }
2349                         break;
2350
2351                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2352
2353                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2354                         var_to_reg_int(s2, src, REG_ITMP2);
2355                         if (iptr->op1 == 0) {
2356                                 gen_nullptr_check(s1);
2357                                 gen_bound_check;
2358                         }
2359                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2360                         break;
2361
2362                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2363
2364                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2365                         var_to_reg_int(s2, src, REG_ITMP2);
2366                         if (iptr->op1 == 0) {
2367                                 gen_nullptr_check(s1);
2368                                 gen_bound_check;
2369                         }
2370                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2371                         break;
2372
2373                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2374
2375                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2376                         var_to_reg_int(s2, src, REG_ITMP2);
2377                         if (iptr->op1 == 0) {
2378                                 gen_nullptr_check(s1);
2379                                 gen_bound_check;
2380                         }
2381                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2382                         break;
2383
2384                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2385
2386                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2387                         var_to_reg_int(s2, src, REG_ITMP2);
2388                         if (iptr->op1 == 0) {
2389                                 gen_nullptr_check(s1);
2390                                 gen_bound_check;
2391                         }
2392                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2393                         break;
2394
2395
2396                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2397                                       /* op1 = type, val.a = field address            */
2398
2399                         if (!iptr->val.a) {
2400                                 codegen_addpatchref(cd, cd->mcodeptr,
2401                                                                         PATCHER_get_putstatic,
2402                                                                         (unresolved_field *) iptr->target);
2403
2404                                 if (showdisassemble) {
2405                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2406                                 }
2407
2408                                 a = 0;
2409
2410                         } else {
2411                                 fieldinfo *fi = iptr->val.a;
2412
2413                                 if (!fi->class->initialized) {
2414                                         codegen_addpatchref(cd, cd->mcodeptr,
2415                                                                                 PATCHER_clinit, fi->class);
2416
2417                                         if (showdisassemble) {
2418                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2419                                         }
2420                                 }
2421
2422                                 a = (ptrint) &(fi->value);
2423                         }
2424
2425                         /* This approach is much faster than moving the field address     */
2426                         /* inline into a register. */
2427                         a = dseg_addaddress(cd, a);
2428                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP2);
2429                         switch (iptr->op1) {
2430                         case TYPE_INT:
2431                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2432                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2433                                 store_reg_to_var_int(iptr->dst, d);
2434                                 break;
2435                         case TYPE_LNG:
2436                         case TYPE_ADR:
2437                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2438                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2439                                 store_reg_to_var_int(iptr->dst, d);
2440                                 break;
2441                         case TYPE_FLT:
2442                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2443                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2444                                 store_reg_to_var_flt(iptr->dst, d);
2445                                 break;
2446                         case TYPE_DBL:
2447                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2448                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2449                                 store_reg_to_var_flt(iptr->dst, d);
2450                                 break;
2451                         }
2452                         break;
2453
2454                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2455                                       /* op1 = type, val.a = field address            */
2456
2457                         if (!iptr->val.a) {
2458                                 codegen_addpatchref(cd, cd->mcodeptr,
2459                                                                         PATCHER_get_putstatic,
2460                                                                         (unresolved_field *) iptr->target);
2461
2462                                 if (showdisassemble) {
2463                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2464                                 }
2465
2466                                 a = 0;
2467
2468                         } else {
2469                                 fieldinfo *fi = iptr->val.a;
2470
2471                                 if (!fi->class->initialized) {
2472                                         codegen_addpatchref(cd, cd->mcodeptr,
2473                                                                                 PATCHER_clinit, fi->class);
2474
2475                                         if (showdisassemble) {
2476                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2477                                         }
2478                                 }
2479
2480                                 a = (ptrint) &(fi->value);
2481                         }
2482
2483                         /* This approach is much faster than moving the field address     */
2484                         /* inline into a register. */
2485                         a = dseg_addaddress(cd, a);
2486                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP2);
2487                         switch (iptr->op1) {
2488                         case TYPE_INT:
2489                                 var_to_reg_int(s2, src, REG_ITMP1);
2490                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2491                                 break;
2492                         case TYPE_LNG:
2493                         case TYPE_ADR:
2494                                 var_to_reg_int(s2, src, REG_ITMP1);
2495                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2496                                 break;
2497                         case TYPE_FLT:
2498                                 var_to_reg_flt(s2, src, REG_FTMP1);
2499                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2500                                 break;
2501                         case TYPE_DBL:
2502                                 var_to_reg_flt(s2, src, REG_FTMP1);
2503                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2504                                 break;
2505                         }
2506                         break;
2507
2508                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2509                                           /* val = value (in current instruction)     */
2510                                           /* op1 = type, val.a = field address (in    */
2511                                           /* following NOP)                           */
2512
2513                         if (!iptr[1].val.a) {
2514                                 codegen_addpatchref(cd, cd->mcodeptr,
2515                                                                         PATCHER_get_putstatic,
2516                                                                         (unresolved_field *) iptr[1].target);
2517
2518                                 if (showdisassemble) {
2519                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2520                                 }
2521
2522                                 a = 0;
2523
2524                         } else {
2525                                 fieldinfo *fi = iptr[1].val.a;
2526
2527                                 if (!fi->class->initialized) {
2528                                         codegen_addpatchref(cd, cd->mcodeptr,
2529                                                                                 PATCHER_clinit, fi->class);
2530
2531                                         if (showdisassemble) {
2532                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2533                                         }
2534                                 }
2535
2536                                 a = (ptrint) &(fi->value);
2537                         }
2538
2539                         /* This approach is much faster than moving the field address     */
2540                         /* inline into a register. */
2541                         a = dseg_addaddress(cd, a);
2542                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP1);
2543                         switch (iptr->op1) {
2544                         case TYPE_INT:
2545                         case TYPE_FLT:
2546                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2547                                 break;
2548                         case TYPE_LNG:
2549                         case TYPE_ADR:
2550                         case TYPE_DBL:
2551                                 if (IS_IMM32(iptr->val.l)) {
2552                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2553                                 } else {
2554                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2555                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2556                                 }
2557                                 break;
2558                         }
2559                         break;
2560
2561                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2562                                       /* op1 = type, val.i = field offset             */
2563
2564                         var_to_reg_int(s1, src, REG_ITMP1);
2565                         gen_nullptr_check(s1);
2566
2567                         if (!iptr->val.a) {
2568                                 codegen_addpatchref(cd, cd->mcodeptr,
2569                                                                         PATCHER_get_putfield,
2570                                                                         (unresolved_field *) iptr->target);
2571
2572                                 if (showdisassemble) {
2573                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2574                                 }
2575
2576                                 a = 0;
2577
2578                         } else {
2579                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2580                         }
2581
2582                         switch (iptr->op1) {
2583                         case TYPE_INT:
2584                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2585                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2586                                 store_reg_to_var_int(iptr->dst, d);
2587                                 break;
2588                         case TYPE_LNG:
2589                         case TYPE_ADR:
2590                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2591                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2592                                 store_reg_to_var_int(iptr->dst, d);
2593                                 break;
2594                         case TYPE_FLT:
2595                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2596                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2597                                 store_reg_to_var_flt(iptr->dst, d);
2598                                 break;
2599                         case TYPE_DBL:
2600                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2601                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2602                                 store_reg_to_var_flt(iptr->dst, d);
2603                                 break;
2604                         }
2605                         break;
2606
2607                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2608                                       /* op1 = type, val.i = field offset             */
2609
2610                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2611                         gen_nullptr_check(s1);
2612                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2613                                 var_to_reg_int(s2, src, REG_ITMP2);
2614                         } else {
2615                                 var_to_reg_flt(s2, src, REG_FTMP2);
2616                         }
2617
2618                         if (!iptr->val.a) {
2619                                 codegen_addpatchref(cd, cd->mcodeptr,
2620                                                                         PATCHER_get_putfield,
2621                                                                         (unresolved_field *) iptr->target);
2622
2623                                 if (showdisassemble) {
2624                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2625                                 }
2626
2627                                 a = 0;
2628
2629                         } else {
2630                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2631                         }
2632
2633                         switch (iptr->op1) {
2634                         case TYPE_INT:
2635                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2636                                 break;
2637                         case TYPE_LNG:
2638                         case TYPE_ADR:
2639                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2640                                 break;
2641                         case TYPE_FLT:
2642                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2643                                 break;
2644                         case TYPE_DBL:
2645                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2646                                 break;
2647                         }
2648                         break;
2649
2650                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2651                                           /* val = value (in current instruction)     */
2652                                           /* op1 = type, val.a = field address (in    */
2653                                           /* following NOP)                           */
2654
2655                         var_to_reg_int(s1, src, REG_ITMP1);
2656                         gen_nullptr_check(s1);
2657
2658                         if (!iptr[1].val.a) {
2659                                 codegen_addpatchref(cd, cd->mcodeptr,
2660                                                                         PATCHER_putfieldconst,
2661                                                                         (unresolved_field *) iptr[1].target);
2662
2663                                 if (showdisassemble) {
2664                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2665                                 }
2666
2667                                 a = 0;
2668
2669                         } else {
2670                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2671                         }
2672
2673                         switch (iptr->op1) {
2674                         case TYPE_INT:
2675                         case TYPE_FLT:
2676                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2677                                 break;
2678                         case TYPE_LNG:
2679                         case TYPE_ADR:
2680                         case TYPE_DBL:
2681                                 /* We can only optimize the move, if the class is resolved.   */
2682                                 /* Otherwise we don't know what to patch.                     */
2683                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2684                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2685                                 } else {
2686                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2687                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2688                                 }
2689                                 break;
2690                         }
2691                         break;
2692
2693
2694                 /* branch operations **************************************************/
2695
2696                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2697
2698                         var_to_reg_int(s1, src, REG_ITMP1);
2699                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2700
2701                         x86_64_call_imm(cd, 0); /* passing exception pointer              */
2702                         x86_64_pop_reg(cd, REG_ITMP2_XPC);
2703
2704                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
2705                         x86_64_jmp_reg(cd, REG_ITMP3);
2706                         break;
2707
2708                 case ICMD_GOTO:         /* ... ==> ...                                */
2709                                         /* op1 = target JavaVM pc                     */
2710
2711                         x86_64_jmp_imm(cd, 0);
2712                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2713                         break;
2714
2715                 case ICMD_JSR:          /* ... ==> ...                                */
2716                                         /* op1 = target JavaVM pc                     */
2717
2718                         x86_64_call_imm(cd, 0);
2719                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2720                         break;
2721
2722                 case ICMD_RET:          /* ... ==> ...                                */
2723                                         /* op1 = local variable                       */
2724
2725                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2726                         var_to_reg_int(s1, var, REG_ITMP1);
2727                         x86_64_jmp_reg(cd, s1);
2728                         break;
2729
2730                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2731                                         /* op1 = target JavaVM pc                     */
2732
2733                         if (src->flags & INMEMORY) {
2734                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2735
2736                         } else {
2737                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2738                         }
2739                         x86_64_jcc(cd, X86_64_CC_E, 0);
2740                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2741                         break;
2742
2743                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2744                                         /* op1 = target JavaVM pc                     */
2745
2746                         if (src->flags & INMEMORY) {
2747                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2748
2749                         } else {
2750                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2751                         }
2752                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2753                         codegen_addreference(cd, BlockPtrOfPC(iptr->op1), cd->mcodeptr);
2754                         break;
2755
2756                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2757                                         /* op1 = target JavaVM pc, val.i = constant   */
2758
2759                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2760                         break;
2761
2762                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2763                                         /* op1 = target JavaVM pc, val.i = constant   */
2764
2765                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2766                         break;
2767
2768                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2769                                         /* op1 = target JavaVM pc, val.i = constant   */
2770
2771                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2772                         break;
2773
2774                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2775                                         /* op1 = target JavaVM pc, val.i = constant   */
2776
2777                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2778                         break;
2779
2780                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2781                                         /* op1 = target JavaVM pc, val.i = constant   */
2782
2783                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2784                         break;
2785
2786                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2787                                         /* op1 = target JavaVM pc, val.i = constant   */
2788
2789                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2790                         break;
2791
2792                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2793                                         /* op1 = target JavaVM pc, val.l = constant   */
2794
2795                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2796                         break;
2797
2798                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2799                                         /* op1 = target JavaVM pc, val.l = constant   */
2800
2801                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2802                         break;
2803
2804                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2805                                         /* op1 = target JavaVM pc, val.l = constant   */
2806
2807                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2808                         break;
2809
2810                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2811                                         /* op1 = target JavaVM pc, val.l = constant   */
2812
2813                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2814                         break;
2815
2816                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2817                                         /* op1 = target JavaVM pc, val.l = constant   */
2818
2819                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2820                         break;
2821
2822                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2823                                         /* op1 = target JavaVM pc, val.l = constant   */
2824
2825                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2826                         break;
2827
2828                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2829                                         /* op1 = target JavaVM pc                     */
2830
2831                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2832                         break;
2833
2834                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2835                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2836
2837                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2838                         break;
2839
2840                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2841                                         /* op1 = target JavaVM pc                     */
2842
2843                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2844                         break;
2845
2846                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2847                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2848
2849                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2850                         break;
2851
2852                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2853                                         /* op1 = target JavaVM pc                     */
2854
2855                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2856                         break;
2857
2858                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2859                                     /* op1 = target JavaVM pc                     */
2860
2861                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2862                         break;
2863
2864                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2865                                         /* op1 = target JavaVM pc                     */
2866
2867                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2868                         break;
2869
2870                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2871                                 /* op1 = target JavaVM pc                     */
2872
2873                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2874                         break;
2875
2876                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2877                                         /* op1 = target JavaVM pc                     */
2878
2879                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2880                         break;
2881
2882                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2883                                         /* op1 = target JavaVM pc                     */
2884
2885                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2886                         break;
2887
2888                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2889                                         /* op1 = target JavaVM pc                     */
2890
2891                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2892                         break;
2893
2894                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2895                                     /* op1 = target JavaVM pc                     */
2896
2897                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2898                         break;
2899
2900                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2901
2902                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2903                         break;
2904
2905                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2906                                         /* val.i = constant                           */
2907
2908                         var_to_reg_int(s1, src, REG_ITMP1);
2909                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2910                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2911                                 if (s1 == d) {
2912                                         M_INTMOVE(s1, REG_ITMP1);
2913                                         s1 = REG_ITMP1;
2914                                 }
2915                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2916                         }
2917                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2918                         x86_64_testl_reg_reg(cd, s1, s1);
2919                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2920                         store_reg_to_var_int(iptr->dst, d);
2921                         break;
2922
2923                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2924                                         /* val.i = constant                           */
2925
2926                         var_to_reg_int(s1, src, REG_ITMP1);
2927                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2928                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2929                                 if (s1 == d) {
2930                                         M_INTMOVE(s1, REG_ITMP1);
2931                                         s1 = REG_ITMP1;
2932                                 }
2933                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2934                         }
2935                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2936                         x86_64_testl_reg_reg(cd, s1, s1);
2937                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2938                         store_reg_to_var_int(iptr->dst, d);
2939                         break;
2940
2941                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2942                                         /* val.i = constant                           */
2943
2944                         var_to_reg_int(s1, src, REG_ITMP1);
2945                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2946                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2947                                 if (s1 == d) {
2948                                         M_INTMOVE(s1, REG_ITMP1);
2949                                         s1 = REG_ITMP1;
2950                                 }
2951                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2952                         }
2953                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2954                         x86_64_testl_reg_reg(cd, s1, s1);
2955                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2956                         store_reg_to_var_int(iptr->dst, d);
2957                         break;
2958
2959                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2960                                         /* val.i = constant                           */
2961
2962                         var_to_reg_int(s1, src, REG_ITMP1);
2963                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2964                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2965                                 if (s1 == d) {
2966                                         M_INTMOVE(s1, REG_ITMP1);
2967                                         s1 = REG_ITMP1;
2968                                 }
2969                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2970                         }
2971                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2972                         x86_64_testl_reg_reg(cd, s1, s1);
2973                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2974                         store_reg_to_var_int(iptr->dst, d);
2975                         break;
2976
2977                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2978                                         /* val.i = constant                           */
2979
2980                         var_to_reg_int(s1, src, REG_ITMP1);
2981                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2982                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2983                                 if (s1 == d) {
2984                                         M_INTMOVE(s1, REG_ITMP1);
2985                                         s1 = REG_ITMP1;
2986                                 }
2987                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2988                         }
2989                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2990                         x86_64_testl_reg_reg(cd, s1, s1);
2991                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2992                         store_reg_to_var_int(iptr->dst, d);
2993                         break;
2994
2995                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2996                                         /* val.i = constant                           */
2997
2998                         var_to_reg_int(s1, src, REG_ITMP1);
2999                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3000                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
3001                                 if (s1 == d) {
3002                                         M_INTMOVE(s1, REG_ITMP1);
3003                                         s1 = REG_ITMP1;
3004                                 }
3005                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
3006                         }
3007                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
3008                         x86_64_testl_reg_reg(cd, s1, s1);
3009                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
3010                         store_reg_to_var_int(iptr->dst, d);
3011                         break;
3012
3013
3014                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
3015                 case ICMD_LRETURN:
3016                 case ICMD_ARETURN:
3017
3018                         var_to_reg_int(s1, src, REG_RESULT);
3019                         M_INTMOVE(s1, REG_RESULT);
3020
3021                         goto nowperformreturn;
3022
3023                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
3024                 case ICMD_DRETURN:
3025
3026                         var_to_reg_flt(s1, src, REG_FRESULT);
3027                         M_FLTMOVE(s1, REG_FRESULT);
3028
3029                         goto nowperformreturn;
3030
3031                 case ICMD_RETURN:      /* ...  ==> ...                                */
3032
3033 nowperformreturn:
3034                         {
3035                         s4 i, p;
3036
3037                         p = parentargs_base;
3038
3039                         /* call trace function */
3040                         if (runverbose) {
3041                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
3042
3043                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
3044                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
3045
3046                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
3047                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
3048                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
3049                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
3050
3051                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
3052                                 x86_64_call_reg(cd, REG_ITMP1);
3053
3054                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
3055                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
3056
3057                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3058                         }
3059
3060 #if defined(USE_THREADS)
3061                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
3062                                 x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, rd->argintregs[0]);
3063
3064                                 /* we need to save the proper return value */
3065                                 switch (iptr->opc) {
3066                                 case ICMD_IRETURN:
3067                                 case ICMD_ARETURN:
3068                                 case ICMD_LRETURN:
3069                                         x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, rd->maxmemuse * 8);
3070                                         break;
3071                                 case ICMD_FRETURN:
3072                                 case ICMD_DRETURN:
3073                                         x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, rd->maxmemuse * 8);
3074                                         break;
3075                                 }
3076
3077                                 x86_64_mov_imm_reg(cd, (ptrint) builtin_monitorexit, REG_ITMP1);
3078                                 x86_64_call_reg(cd, REG_ITMP1);
3079
3080                                 /* and now restore the proper return value */
3081                                 switch (iptr->opc) {
3082                                 case ICMD_IRETURN:
3083                                 case ICMD_ARETURN:
3084                                 case ICMD_LRETURN:
3085                                         x86_64_mov_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_RESULT);
3086                                         break;
3087                                 case ICMD_FRETURN:
3088                                 case ICMD_DRETURN:
3089                                         x86_64_movq_membase_reg(cd, REG_SP, rd->maxmemuse * 8, REG_FRESULT);
3090                                         break;
3091                                 }
3092                         }
3093 #endif
3094
3095                         /* restore saved registers                                        */
3096                         for (i = rd->savintregcnt - 1; i >= rd->maxsavintreguse; i--) {
3097                                 p--; x86_64_mov_membase_reg(cd, REG_SP, p * 8, rd->savintregs[i]);
3098                         }
3099                         for (i = rd->savfltregcnt - 1; i >= rd->maxsavfltreguse; i--) {
3100                                 p--; x86_64_movq_membase_reg(cd, REG_SP, p * 8, rd->savfltregs[i]);
3101                         }
3102
3103                         /* deallocate stack                                               */
3104                         if (parentargs_base) {
3105                                 x86_64_alu_imm_reg(cd, X86_64_ADD, parentargs_base * 8, REG_SP);
3106                         }
3107
3108                         x86_64_ret(cd);
3109                         }
3110                         break;
3111
3112
3113                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3114                         {
3115                                 s4 i, l, *s4ptr;
3116                                 void **tptr;
3117
3118                                 tptr = (void **) iptr->target;
3119
3120                                 s4ptr = iptr->val.a;
3121                                 l = s4ptr[1];                          /* low     */
3122                                 i = s4ptr[2];                          /* high    */
3123
3124                                 var_to_reg_int(s1, src, REG_ITMP1);
3125                                 M_INTMOVE(s1, REG_ITMP1);
3126                                 if (l != 0) {
3127                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3128                                 }
3129                                 i = i - l + 1;
3130
3131                 /* range check */
3132                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3133                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3134
3135                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3136                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3137
3138                                 /* build jump table top down and use address of lowest entry */
3139
3140                 /* s4ptr += 3 + i; */
3141                                 tptr += i;
3142
3143                                 while (--i >= 0) {
3144                                         /* dseg_addtarget(cd, BlockPtrOfPC(*--s4ptr)); */
3145                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3146                                         --tptr;
3147                                 }
3148
3149                                 /* length of dataseg after last dseg_addtarget is used by load */
3150
3151                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3152                                 dseg_adddata(cd, cd->mcodeptr);
3153                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3154                                 x86_64_jmp_reg(cd, REG_ITMP1);
3155                         }
3156                         break;
3157
3158
3159                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3160                         {
3161                                 s4 i, l, val, *s4ptr;
3162                                 void **tptr;
3163
3164                                 tptr = (void **) iptr->target;
3165
3166                                 s4ptr = iptr->val.a;
3167                                 l = s4ptr[0];                          /* default  */
3168                                 i = s4ptr[1];                          /* count    */
3169
3170                                 MCODECHECK((i<<2)+8);
3171                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3172                                 while (--i >= 0) {
3173                                         s4ptr += 2;
3174                                         ++tptr;
3175
3176                                         val = s4ptr[0];
3177                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3178                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3179                                         /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[1]), cd->mcodeptr); */
3180                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3181                                 }
3182
3183                                 x86_64_jmp_imm(cd, 0);
3184                                 /* codegen_addreference(cd, BlockPtrOfPC(l), cd->mcodeptr); */
3185
3186                                 tptr = (void **) iptr->target;
3187                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3188                         }
3189                         break;
3190
3191
3192                 case ICMD_BUILTIN3:     /* ..., arg1, arg2, arg3 ==> ...              */
3193                                         /* op1 = return type, val.a = function pointer*/
3194                         s3 = 3;
3195                         goto gen_method;
3196
3197                 case ICMD_BUILTIN2:     /* ..., arg1, arg2 ==> ...                    */
3198                                         /* op1 = return type, val.a = function pointer*/
3199                         s3 = 2;
3200                         goto gen_method;
3201
3202                 case ICMD_BUILTIN1:     /* ..., arg1 ==> ...                          */
3203                                         /* op1 = return type, val.a = function pointer*/
3204                         s3 = 1;
3205                         goto gen_method;
3206
3207                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3208                                         /* op1 = arg count, val.a = method pointer    */
3209
3210                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3211                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3212                 case ICMD_INVOKEINTERFACE:
3213
3214                         s3 = iptr->op1;
3215
3216 gen_method: {
3217                         methodinfo *lm;
3218                         stackptr    tmpsrc;
3219                         s4          iarg;
3220                         s4          farg;
3221
3222                         MCODECHECK((s3 << 1) + 64);
3223
3224                         /* copy arguments to registers or stack location ******************/
3225
3226                         /* count integer and float arguments */
3227
3228                         iarg = 0;
3229                         farg = 0;
3230
3231                         for (s2 = s3, tmpsrc = src; --s2 >= 0; tmpsrc = tmpsrc->prev) {
3232                                 IS_INT_LNG_TYPE(tmpsrc->type) ? iarg++ : farg++;
3233                         }
3234
3235                         /* calculate amount of arguments to be on stack */
3236
3237                         s2 = (iarg > INT_ARG_CNT) ? iarg - INT_ARG_CNT : 0 +
3238                                 (farg > FLT_ARG_CNT) ? farg - FLT_ARG_CNT : 0;
3239
3240                         for (; --s3 >= 0; src = src->prev) {
3241                                 /* decrement the current argument type */
3242                                 IS_INT_LNG_TYPE(src->type) ? iarg-- : farg--;
3243
3244                                 if (src->varkind == ARGVAR) {
3245                                         if (IS_INT_LNG_TYPE(src->type)) {
3246                                                 if (iarg >= INT_ARG_CNT) {
3247                                                         s2--;
3248                                                 }
3249                                         } else {
3250                                                 if (farg >= FLT_ARG_CNT) {
3251                                                         s2--;
3252                                                 }
3253                                         }
3254                                         continue;
3255                                 }
3256
3257                                 if (IS_INT_LNG_TYPE(src->type)) {
3258                                         if (iarg < INT_ARG_CNT) {
3259                                                 s1 = rd->argintregs[iarg];
3260                                                 var_to_reg_int(d, src, s1);
3261                                                 M_INTMOVE(d, s1);
3262
3263                                         } else {
3264                                                 var_to_reg_int(d, src, REG_ITMP1);
3265                                                 s2--;
3266                                                 x86_64_mov_reg_membase(cd, d, REG_SP, s2 * 8);
3267                                         }
3268
3269                                 } else {
3270                                         if (farg < FLT_ARG_CNT) {
3271                                                 s1 = rd->argfltregs[farg];
3272                                                 var_to_reg_flt(d, src, s1);
3273                                                 M_FLTMOVE(d, s1);
3274
3275                                         } else {
3276                                                 var_to_reg_flt(d, src, REG_FTMP1);
3277                                                 s2--;
3278                                                 x86_64_movq_reg_membase(cd, d, REG_SP, s2 * 8);
3279                                         }
3280                                 }
3281                         } /* end of for */
3282
3283                         lm = iptr->val.a;
3284                         switch (iptr->opc) {
3285                         case ICMD_BUILTIN3:
3286                         case ICMD_BUILTIN2:
3287                         case ICMD_BUILTIN1:
3288                                 d = iptr->op1;
3289
3290                                 if (iptr->target) {
3291                                         codegen_addpatchref(cd, cd->mcodeptr,
3292                                                                                 iptr->val.fp, iptr->target);
3293
3294                                         if (showdisassemble) {
3295                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3296                                         }
3297
3298                                         a = 0;
3299
3300                                 } else {
3301                                         a = (ptrint) lm;
3302                                 }
3303
3304                                 x86_64_mov_imm_reg(cd, a, REG_ITMP1);
3305                                 x86_64_call_reg(cd, REG_ITMP1);
3306                                 break;
3307
3308                         case ICMD_INVOKESPECIAL:
3309                                 x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
3310                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
3311                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3312
3313                                 /* first argument contains pointer */
3314 /*                              gen_nullptr_check(rd->argintregs[0]); */
3315
3316                                 /* access memory for hardware nullptr */
3317 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3318
3319                                 /* fall through */
3320
3321                         case ICMD_INVOKESTATIC:
3322                                 if (!lm) {
3323                                         unresolved_method *um = iptr->target;
3324
3325                                         codegen_addpatchref(cd, cd->mcodeptr,
3326                                                                                 PATCHER_invokestatic_special, um);
3327
3328                                         if (showdisassemble) {
3329                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3330                                         }
3331
3332                                         a = 0;
3333                                         d = um->methodref->parseddesc.md->returntype.type;
3334
3335                                 } else {
3336                                         a = (ptrint) lm->stubroutine;
3337                                         d = lm->parseddesc->returntype.type;
3338                                 }
3339
3340                                 x86_64_mov_imm_reg(cd, a, REG_ITMP2);
3341                                 x86_64_call_reg(cd, REG_ITMP2);
3342                                 break;
3343
3344                         case ICMD_INVOKEVIRTUAL:
3345                                 gen_nullptr_check(rd->argintregs[0]);
3346
3347                                 if (!lm) {
3348                                         unresolved_method *um = iptr->target;
3349
3350                                         codegen_addpatchref(cd, cd->mcodeptr,
3351                                                                                 PATCHER_invokevirtual, um);
3352
3353                                         if (showdisassemble) {
3354                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3355                                         }
3356
3357                                         s1 = 0;
3358                                         d = um->methodref->parseddesc.md->returntype.type;
3359
3360                                 } else {
3361                                         s1 = OFFSET(vftbl_t, table[0]) +
3362                                                 sizeof(methodptr) * lm->vftblindex;
3363                                         d = lm->parseddesc->returntype.type;
3364                                 }
3365
3366                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3367                                                                            OFFSET(java_objectheader, vftbl),
3368                                                                            REG_ITMP2);
3369                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3370                                 x86_64_call_reg(cd, REG_ITMP1);
3371                                 break;
3372
3373                         case ICMD_INVOKEINTERFACE:
3374                                 gen_nullptr_check(rd->argintregs[0]);
3375
3376                                 if (!lm) {
3377                                         unresolved_method *um = iptr->target;
3378
3379                                         codegen_addpatchref(cd, cd->mcodeptr,
3380                                                                                 PATCHER_invokeinterface, um);
3381
3382                                         if (showdisassemble) {
3383                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3384                                         }
3385
3386                                         s1 = 0;
3387                                         s2 = 0;
3388                                         d = um->methodref->parseddesc.md->returntype.type;
3389
3390                                 } else {
3391                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3392                                                 sizeof(methodptr) * lm->class->index;
3393
3394                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3395
3396                                         d = lm->parseddesc->returntype.type;
3397                                 }
3398
3399                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3400                                                                            OFFSET(java_objectheader, vftbl),
3401                                                                            REG_ITMP2);
3402                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3403                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3404                                 x86_64_call_reg(cd, REG_ITMP1);
3405                                 break;
3406                         }
3407
3408                         /* d contains return type */
3409
3410                         if (d != TYPE_VOID) {
3411                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3412                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3413                                         M_INTMOVE(REG_RESULT, s1);
3414                                         store_reg_to_var_int(iptr->dst, s1);
3415
3416                                 } else {
3417                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3418                                         M_FLTMOVE(REG_FRESULT, s1);
3419                                         store_reg_to_var_flt(iptr->dst, s1);
3420                                 }
3421                         }
3422                         }
3423                         break;
3424
3425
3426                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3427
3428                                       /* op1:   0 == array, 1 == class                */
3429                                       /* val.a: (classinfo *) superclass              */
3430
3431                         /*  superclass is an interface:
3432                          *
3433                          *  OK if ((sub == NULL) ||
3434                          *         (sub->vftbl->interfacetablelength > super->index) &&
3435                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3436                          *
3437                          *  superclass is a class:
3438                          *
3439                          *  OK if ((sub == NULL) || (0
3440                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3441                          *         super->vftbl->diffval));
3442                          */
3443
3444                         {
3445                         classinfo *super;
3446                         vftbl_t   *supervftbl;
3447                         s4         superindex;
3448
3449                         super = (classinfo *) iptr->val.a;
3450
3451                         if (!super) {
3452                                 superindex = 0;
3453                                 supervftbl = NULL;
3454
3455                         } else {
3456                                 superindex = super->index;
3457                                 supervftbl = super->vftbl;
3458                         }
3459
3460 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3461             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3462 #endif
3463                         var_to_reg_int(s1, src, REG_ITMP1);
3464
3465                         /* calculate interface checkcast code size */
3466
3467                         s2 = 3; /* mov_membase_reg */
3468                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3469
3470                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3471                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3472                                 3 /* test */ + 6 /* jcc */;
3473
3474                         if (!super)
3475                                 s2 += (showdisassemble ? 5 : 0);
3476
3477                         /* calculate class checkcast code size */
3478
3479                         s3 = 3; /* mov_membase_reg */
3480                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3481                         s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3482
3483 #if 0
3484                         if (s1 != REG_ITMP1) {
3485                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3486                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3487                                 a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3488                                 CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3489                                 a += 3;    /* sub */
3490
3491                         } else
3492 #endif
3493                                 {
3494                                         s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3495                                                 10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3496                                         CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3497                                 }
3498
3499                         s3 += 3 /* cmp */ + 6 /* jcc */;
3500
3501                         if (!super)
3502                                 s3 += (showdisassemble ? 5 : 0);
3503
3504                         /* if class is not resolved, check which code to call */
3505
3506                         if (!super) {
3507                                 x86_64_test_reg_reg(cd, s1, s1);
3508                                 x86_64_jcc(cd, X86_64_CC_Z, 6 + (showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3509
3510                                 codegen_addpatchref(cd, cd->mcodeptr,
3511                                                                         PATCHER_checkcast_instanceof_flags,
3512                                                                         (constant_classref *) iptr->target);
3513
3514                                 if (showdisassemble) {
3515                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3516                                 }
3517
3518                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP2); /* super->flags */
3519                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP2);
3520                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3521                         }
3522
3523                         /* interface checkcast code */
3524
3525                         if (!super || (super->flags & ACC_INTERFACE)) {
3526                                 if (super) {
3527                                         x86_64_test_reg_reg(cd, s1, s1);
3528                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3529                                 }
3530
3531                                 x86_64_mov_membase_reg(cd, s1,
3532                                                                            OFFSET(java_objectheader, vftbl),
3533                                                                            REG_ITMP2);
3534
3535                                 if (!super) {
3536                                         codegen_addpatchref(cd, cd->mcodeptr,
3537                                                                                 PATCHER_checkcast_instanceof_interface,
3538                                                                                 (constant_classref *) iptr->target);
3539
3540                                         if (showdisassemble) {
3541                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3542                                         }
3543                                 }
3544
3545                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3546                                                                                   OFFSET(vftbl_t, interfacetablelength),
3547                                                                                   REG_ITMP3);
3548                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3549                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3550                                 x86_64_jcc(cd, X86_64_CC_LE, 0);
3551                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3552                                 x86_64_mov_membase32_reg(cd, REG_ITMP2,
3553                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3554                                                                                  superindex * sizeof(methodptr*),
3555                                                                                  REG_ITMP3);
3556                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3557                                 x86_64_jcc(cd, X86_64_CC_E, 0);
3558                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3559
3560                                 if (!super)
3561                                         x86_64_jmp_imm(cd, s3);
3562                         }
3563
3564                         /* class checkcast code */
3565
3566                         if (!super || !(super->flags & ACC_INTERFACE)) {
3567                                 if (super) {
3568                                         x86_64_test_reg_reg(cd, s1, s1);
3569                                         x86_64_jcc(cd, X86_64_CC_Z, s3);
3570                                 }
3571
3572                                 x86_64_mov_membase_reg(cd, s1,
3573                                                                            OFFSET(java_objectheader, vftbl),
3574                                                                            REG_ITMP2);
3575
3576                                 if (!super) {
3577                                         codegen_addpatchref(cd, cd->mcodeptr,
3578                                                                                 PATCHER_checkcast_class,
3579                                                                                 (constant_classref *) iptr->target);
3580
3581                                         if (showdisassemble) {
3582                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3583                                         }
3584                                 }
3585
3586                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3587 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3588                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3589 #endif
3590                                 x86_64_movl_membase32_reg(cd, REG_ITMP2,
3591                                                                                   OFFSET(vftbl_t, baseval),
3592                                                                                   REG_ITMP2);
3593 /*                                      if (s1 != REG_ITMP1) { */
3594 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3595 /*                                                                                              OFFSET(vftbl_t, baseval), */
3596 /*                                                                                              REG_ITMP1); */
3597 /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3598 /*                                                                                              OFFSET(vftbl_t, diffval), */
3599 /*                                                                                              REG_ITMP3); */
3600 /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3601 /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3602 /*  #endif */
3603 /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3604
3605 /*                                      } else { */
3606                                 x86_64_movl_membase32_reg(cd, REG_ITMP3,
3607                                                                                   OFFSET(vftbl_t, baseval),
3608                                                                                   REG_ITMP3);
3609                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP3, REG_ITMP2);
3610                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP3);
3611                                 x86_64_movl_membase_reg(cd, REG_ITMP3,
3612                                                                                 OFFSET(vftbl_t, diffval),
3613                                                                                 REG_ITMP3);
3614 /*                                      } */
3615 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3616                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3617 #endif
3618                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP2);
3619                                 x86_64_jcc(cd, X86_64_CC_A, 0);    /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3620                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3621                         }
3622                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3623                         M_INTMOVE(s1, d);
3624                         store_reg_to_var_int(iptr->dst, d);
3625 /*                      if (iptr->dst->flags & INMEMORY) { */
3626 /*                              x86_64_mov_reg_membase(cd, s1, REG_SP, iptr->dst->regoff * 8); */
3627 /*                      } else { */
3628 /*                              M_INTMOVE(s1, iptr->dst->regoff); */
3629 /*                      } */
3630                         }
3631                         break;
3632
3633                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3634
3635                                       /* op1:   0 == array, 1 == class                */
3636                                       /* val.a: (classinfo *) superclass              */
3637
3638                         /*  superclass is an interface:
3639                          *
3640                          *  return (sub != NULL) &&
3641                          *         (sub->vftbl->interfacetablelength > super->index) &&
3642                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3643                          *
3644                          *  superclass is a class:
3645                          *
3646                          *  return ((sub != NULL) && (0
3647                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3648                          *          super->vftbl->diffvall));
3649                          */
3650
3651                         {
3652                         classinfo *super;
3653                         vftbl_t   *supervftbl;
3654                         s4         superindex;
3655
3656                         super = (classinfo *) iptr->val.a;
3657
3658                         if (!super) {
3659                                 superindex = 0;
3660                                 supervftbl = NULL;
3661
3662                         } else {
3663                                 superindex = super->index;
3664                                 supervftbl = super->vftbl;
3665                         }
3666
3667 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3668             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3669 #endif
3670
3671                         var_to_reg_int(s1, src, REG_ITMP1);
3672                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3673                         if (s1 == d) {
3674                                 M_INTMOVE(s1, REG_ITMP1);
3675                                 s1 = REG_ITMP1;
3676                         }
3677
3678                         /* calculate interface instanceof code size */
3679
3680                         s2 = 3; /* mov_membase_reg */
3681                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3682                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3683                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3684                                 3 /* test */ + 4 /* setcc */;
3685
3686                         if (!super)
3687                                 s2 += (showdisassemble ? 5 : 0);
3688
3689                         /* calculate class instanceof code size */
3690
3691                         s3 = 3; /* mov_membase_reg */
3692                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3693                         s3 += 10; /* mov_imm_reg */
3694                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3695                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3696                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3697                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3698                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3699                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3700                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3701
3702                         if (!super)
3703                                 s3 += (showdisassemble ? 5 : 0);
3704
3705                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3706
3707                         /* if class is not resolved, check which code to call */
3708
3709                         if (!super) {
3710                                 x86_64_test_reg_reg(cd, s1, s1);
3711                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (showdisassemble ? 5 : 0) +
3712                                                                                          7 + 6 + s2 + 5 + s3));
3713
3714                                 codegen_addpatchref(cd, cd->mcodeptr,
3715                                                                         PATCHER_checkcast_instanceof_flags,
3716                                                                         (constant_classref *) iptr->target);
3717
3718                                 if (showdisassemble) {
3719                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3720                                 }
3721
3722                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3723                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3724                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3725                         }
3726
3727                         /* interface instanceof code */
3728
3729                         if (!super || (super->flags & ACC_INTERFACE)) {
3730                                 if (super) {
3731                                         x86_64_test_reg_reg(cd, s1, s1);
3732                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3733                                 }
3734
3735                                 x86_64_mov_membase_reg(cd, s1,
3736                                                                            OFFSET(java_objectheader, vftbl),
3737                                                                            REG_ITMP1);
3738                                 if (!super) {
3739                                         codegen_addpatchref(cd, cd->mcodeptr,
3740                                                                                 PATCHER_checkcast_instanceof_interface,
3741                                                                                 (constant_classref *) iptr->target);
3742
3743                                         if (showdisassemble) {
3744                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3745                                         }
3746                                 }
3747
3748                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3749                                                                                   OFFSET(vftbl_t, interfacetablelength),
3750                                                                                   REG_ITMP3);
3751                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3752                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3753
3754                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3755
3756                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3757                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3758                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3759                                                                                  superindex * sizeof(methodptr*),
3760                                                                                  REG_ITMP1);
3761                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3762                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3763
3764                                 if (!super)
3765                                         x86_64_jmp_imm(cd, s3);
3766                         }
3767
3768                         /* class instanceof code */
3769
3770                         if (!super || !(super->flags & ACC_INTERFACE)) {
3771                                 if (super) {
3772                                         x86_64_test_reg_reg(cd, s1, s1);
3773                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3774                                 }
3775
3776                                 x86_64_mov_membase_reg(cd, s1,
3777                                                                            OFFSET(java_objectheader, vftbl),
3778                                                                            REG_ITMP1);
3779
3780                                 if (!super) {
3781                                         codegen_addpatchref(cd, cd->mcodeptr,
3782                                                                                 PATCHER_instanceof_class,
3783                                                                                 (constant_classref *) iptr->target);
3784
3785                                         if (showdisassemble) {
3786                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3787                                         }
3788                                 }
3789
3790                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3791 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3792                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3793 #endif
3794                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3795                                                                                 OFFSET(vftbl_t, baseval),
3796                                                                                 REG_ITMP1);
3797                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3798                                                                                 OFFSET(vftbl_t, diffval),
3799                                                                                 REG_ITMP3);
3800                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3801                                                                                 OFFSET(vftbl_t, baseval),
3802                                                                                 REG_ITMP2);
3803 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3804                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3805 #endif
3806                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3807                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3808                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3809                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3810                         }
3811                         store_reg_to_var_int(iptr->dst, d);
3812                         }
3813                         break;
3814
3815                 case ICMD_CHECKASIZE:  /* ..., size ==> ..., size                     */
3816
3817                         if (src->flags & INMEMORY) {
3818                                 x86_64_alul_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
3819
3820                         } else {
3821                                 x86_64_testl_reg_reg(cd, src->regoff, src->regoff);
3822                         }
3823                         x86_64_jcc(cd, X86_64_CC_L, 0);
3824                         codegen_addxcheckarefs(cd, cd->mcodeptr);
3825                         break;
3826
3827                 case ICMD_CHECKEXCEPTION:    /* ... ==> ...                           */
3828
3829                         x86_64_test_reg_reg(cd, REG_RESULT, REG_RESULT);
3830                         x86_64_jcc(cd, X86_64_CC_E, 0);
3831                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3832                         break;
3833
3834                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3835                                          /* op1 = dimension, val.a = array descriptor */
3836
3837                         /* check for negative sizes and copy sizes to stack if necessary  */
3838
3839                         MCODECHECK((iptr->op1 << 1) + 64);
3840
3841                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3842                                 var_to_reg_int(s2, src, REG_ITMP1);
3843                                 x86_64_testl_reg_reg(cd, s2, s2);
3844                                 x86_64_jcc(cd, X86_64_CC_L, 0);
3845                                 codegen_addxcheckarefs(cd, cd->mcodeptr);
3846
3847                                 /* copy SAVEDVAR sizes to stack */
3848
3849                                 if (src->varkind != ARGVAR) {
3850                                         x86_64_mov_reg_membase(cd, s2, REG_SP, s1 * 8);
3851                                 }
3852                         }
3853
3854                         /* is a patcher function set? */
3855
3856                         if (iptr->target) {
3857                                 codegen_addpatchref(cd, cd->mcodeptr,
3858                                                                         (functionptr) (ptrint) iptr->target,
3859                                                                         iptr->val.a);
3860
3861                                 if (showdisassemble) {
3862                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3863                                 }
3864
3865                                 a = 0;
3866
3867                         } else {
3868                                 a = (ptrint) iptr->val.a;
3869                         }
3870
3871                         /* a0 = dimension count */
3872
3873                         x86_64_mov_imm_reg(cd, iptr->op1, rd->argintregs[0]);
3874
3875                         /* a1 = arrayvftbl */
3876
3877                         x86_64_mov_imm_reg(cd, (ptrint) iptr->val.a, rd->argintregs[1]);
3878
3879                         /* a2 = pointer to dimensions = stack pointer */
3880
3881                         x86_64_mov_reg_reg(cd, REG_SP, rd->argintregs[2]);
3882
3883                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_multianewarray, REG_ITMP1);
3884                         x86_64_call_reg(cd, REG_ITMP1);
3885
3886                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3887                         M_INTMOVE(REG_RESULT, s1);
3888                         store_reg_to_var_int(iptr->dst, s1);
3889                         break;
3890
3891                 default:
3892                         throw_cacao_exception_exit(string_java_lang_InternalError,
3893                                                                            "Unknown ICMD %d", iptr->opc);
3894         } /* switch */
3895
3896         } /* for instruction */
3897
3898         /* copy values to interface registers */
3899
3900         src = bptr->outstack;
3901         len = bptr->outdepth;
3902         MCODECHECK(64 + len);
3903 #ifdef LSRA
3904         if (!opt_lsra)
3905 #endif
3906         while (src) {
3907                 len--;
3908                 if ((src->varkind != STACKVAR)) {
3909                         s2 = src->type;
3910                         if (IS_FLT_DBL_TYPE(s2)) {
3911                                 var_to_reg_flt(s1, src, REG_FTMP1);
3912                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3913                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3914
3915                                 } else {
3916                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3917                                 }
3918
3919                         } else {
3920                                 var_to_reg_int(s1, src, REG_ITMP1);
3921                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3922                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3923
3924                                 } else {
3925                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3926                                 }
3927                         }
3928                 }
3929                 src = src->prev;
3930         }
3931         } /* if (bptr -> flags >= BBREACHED) */
3932         } /* for basic block */
3933
3934         codegen_createlinenumbertable(cd);
3935
3936         {
3937
3938         /* generate bound check stubs */
3939
3940         u1 *xcodeptr = NULL;
3941         branchref *bref;
3942
3943         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3944                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3945                                   bref->branchpos,
3946                                                   cd->mcodeptr - cd->mcodebase);
3947
3948                 MCODECHECK(100);
3949
3950                 /* move index register into REG_ITMP1 */
3951                 x86_64_mov_reg_reg(cd, bref->reg, REG_ITMP1);             /* 3 bytes  */
3952
3953                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
3954                 dseg_adddata(cd, cd->mcodeptr);
3955                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
3956                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
3957
3958                 if (xcodeptr != NULL) {
3959                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
3960
3961                 } else {
3962                         xcodeptr = cd->mcodeptr;
3963
3964
3965                         /*create stackinfo -- begin*/
3966                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
3967                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
3968                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
3969                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
3970                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
3971                         x86_64_call_reg(cd,REG_ITMP3);
3972                         /*create stackinfo -- end*/
3973
3974                         x86_64_mov_reg_reg(cd, REG_ITMP1, rd->argintregs[0]);
3975                         x86_64_mov_imm_reg(cd, (ptrint) new_arrayindexoutofboundsexception, REG_ITMP3);
3976                         x86_64_call_reg(cd, REG_ITMP3);
3977
3978                         /*remove stackinfo -- begin*/
3979                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
3980                         x86_64_call_reg(cd,REG_ITMP3);
3981                         /*remove stackinfo -- end*/
3982
3983                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
3984                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3985
3986                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
3987                         x86_64_jmp_reg(cd, REG_ITMP3);
3988                 }
3989         }
3990
3991         /* generate negative array size check stubs */
3992
3993         xcodeptr = NULL;
3994
3995         for (bref = cd->xcheckarefs; bref != NULL; bref = bref->next) {
3996                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
3997                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
3998                                                           bref->branchpos,
3999                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4000                         continue;
4001                 }
4002
4003                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4004                                   bref->branchpos,
4005                                                   cd->mcodeptr - cd->mcodebase);
4006
4007                 MCODECHECK(100);
4008
4009                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4010                 dseg_adddata(cd, cd->mcodeptr);
4011                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
4012                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
4013
4014                 if (xcodeptr != NULL) {
4015                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4016
4017                 } else {
4018                         xcodeptr = cd->mcodeptr;
4019
4020
4021                         /*create stackinfo -- begin*/
4022                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
4023                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
4024                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
4025                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
4026                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
4027                         x86_64_call_reg(cd,REG_ITMP3);
4028                         /*create stackinfo -- end*/
4029
4030                         x86_64_mov_imm_reg(cd, (u8) new_negativearraysizeexception, REG_ITMP3);
4031                         x86_64_call_reg(cd, REG_ITMP3);
4032
4033                         /*remove stackinfo -- begin*/
4034                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
4035                         x86_64_call_reg(cd,REG_ITMP3);
4036                         /*remove stackinfo -- end*/
4037
4038                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
4039                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
4040
4041                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
4042                         x86_64_jmp_reg(cd, REG_ITMP3);
4043                 }
4044         }
4045
4046         /* generate cast check stubs */
4047
4048         xcodeptr = NULL;
4049
4050         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
4051                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4052                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4053                                                           bref->branchpos,
4054                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4055                         continue;
4056                 }
4057
4058                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4059                                   bref->branchpos,
4060                                                   cd->mcodeptr - cd->mcodebase);
4061
4062                 MCODECHECK(100);
4063
4064                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4065                 dseg_adddata(cd, cd->mcodeptr);
4066                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
4067                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
4068
4069                 if (xcodeptr != NULL) {
4070                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4071
4072                 } else {
4073                         xcodeptr = cd->mcodeptr;
4074
4075                         /*create stackinfo -- begin*/
4076                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
4077                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
4078                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
4079                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
4080                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
4081                         x86_64_call_reg(cd,REG_ITMP3);
4082                         /*create stackinfo -- end*/
4083
4084
4085                         x86_64_mov_imm_reg(cd, (u8) new_classcastexception, REG_ITMP3);
4086                         x86_64_call_reg(cd, REG_ITMP3);
4087
4088                         /*remove stackinfo -- begin*/
4089                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
4090                         x86_64_call_reg(cd,REG_ITMP3);
4091                         /*remove stackinfo -- end*/
4092
4093                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
4094                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
4095
4096                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
4097                         x86_64_jmp_reg(cd, REG_ITMP3);
4098                 }
4099         }
4100
4101         /* generate divide by zero check stubs */
4102
4103         xcodeptr = NULL;
4104
4105         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
4106                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4107                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4108                                                           bref->branchpos,
4109                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4110                         continue;
4111                 }
4112
4113                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4114                                   bref->branchpos,
4115                                                   cd->mcodeptr - cd->mcodebase);
4116
4117                 MCODECHECK(100);
4118
4119                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4120                 dseg_adddata(cd, cd->mcodeptr);
4121                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP3);   /* 10 bytes */
4122                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP3, REG_ITMP2_XPC); /* 3 bytes  */
4123
4124                 if (xcodeptr != NULL) {
4125                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4126
4127                 } else {
4128                         xcodeptr = cd->mcodeptr;
4129
4130                         /*create stackinfo -- begin*/
4131                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
4132                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
4133                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
4134                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
4135                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
4136                         x86_64_call_reg(cd,REG_ITMP3);
4137                         /*create stackinfo -- end*/
4138
4139                         x86_64_mov_imm_reg(cd, (u8) new_arithmeticexception, REG_ITMP3);
4140                         x86_64_call_reg(cd, REG_ITMP3);
4141
4142                         /*remove stackinfo -- begin*/
4143                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
4144                         x86_64_call_reg(cd,REG_ITMP3);
4145                         /*remove stackinfo -- end*/
4146
4147                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
4148                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
4149
4150                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
4151                         x86_64_jmp_reg(cd, REG_ITMP3);
4152                 }
4153         }
4154
4155         /* generate exception check stubs */
4156
4157         xcodeptr = NULL;
4158
4159         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
4160                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4161                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4162                                                           bref->branchpos,
4163                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4164                         continue;
4165                 }
4166
4167                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4168                                   bref->branchpos,
4169                                                   cd->mcodeptr - cd->mcodebase);
4170
4171                 MCODECHECK(100);
4172
4173                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4174                 dseg_adddata(cd, cd->mcodeptr);
4175                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);   /* 10 bytes */
4176                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC); /* 3 bytes  */
4177
4178                 if (xcodeptr != NULL) {
4179                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4180
4181                 } else {
4182                         xcodeptr = cd->mcodeptr;
4183
4184
4185
4186                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4*8, REG_SP);
4187                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3*8);
4188                         x86_64_mov_imm_membase(cd, 0, REG_SP, 2*8);
4189                         x86_64_mov_imm_membase(cd, 0, REG_SP, 1*8);
4190                         x86_64_mov_imm_membase(cd, 0, REG_SP, 0*8);
4191                         x86_64_mov_imm_reg(cd,(u8) asm_prepare_native_stackinfo,REG_ITMP1);
4192                         x86_64_call_reg(cd,REG_ITMP1);
4193
4194
4195 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4196                         x86_64_mov_imm_reg(cd, (u8) &builtin_get_exceptionptrptr, REG_ITMP1);
4197                         x86_64_call_reg(cd, REG_ITMP1);
4198                         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
4199                         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4200                         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4201 #else
4202                         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
4203                         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP1_XPTR);
4204                         x86_64_mov_imm_membase(cd, 0, REG_ITMP3, 0);
4205 #endif
4206                         x86_64_mov_reg_reg(cd,REG_ITMP1_XPTR,RDI);
4207                         x86_64_mov_imm_reg(cd,(u8) helper_fillin_stacktrace_always,REG_ITMP1);
4208                         x86_64_call_reg(cd,REG_ITMP1);
4209                         x86_64_mov_reg_reg(cd,REG_RESULT,REG_ITMP1_XPTR);
4210
4211                         x86_64_mov_imm_reg(cd,(u8) asm_remove_native_stackinfo,REG_ITMP2);
4212                         x86_64_call_reg(cd,REG_ITMP2);
4213
4214                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
4215                         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);
4216                         x86_64_alu_imm_reg(cd, X86_64_ADD, 8, REG_SP);
4217
4218
4219                         x86_64_mov_imm_reg(cd, (u8) asm_handle_exception, REG_ITMP3);
4220                         x86_64_jmp_reg(cd, REG_ITMP3);
4221                 }
4222         }
4223
4224         /* generate null pointer check stubs */
4225
4226         xcodeptr = NULL;
4227
4228         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
4229                 if ((cd->exceptiontablelength == 0) && (xcodeptr != NULL)) {
4230                         gen_resolvebranch(cd->mcodebase + bref->branchpos,
4231                                                           bref->branchpos,
4232                                                           xcodeptr - cd->mcodebase - (10 + 10 + 3));
4233                         continue;
4234                 }
4235
4236                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4237                                   bref->branchpos,
4238                                                   cd->mcodeptr - cd->mcodebase);
4239
4240                 MCODECHECK(100);
4241
4242                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2_XPC);                 /* 10 bytes */
4243                 dseg_adddata(cd, cd->mcodeptr);
4244                 x86_64_mov_imm_reg(cd, bref->branchpos - 6, REG_ITMP1);   /* 10 bytes */
4245                 x86_64_alu_reg_reg(cd, X86_64_ADD, REG_ITMP1, REG_ITMP2_XPC); /* 3 bytes  */
4246
4247                 if (xcodeptr != NULL) {
4248                         x86_64_jmp_imm(cd, xcodeptr - cd->mcodeptr - 5);
4249
4250                 } else {
4251                         xcodeptr = cd->mcodeptr;
4252
4253                         /*create stackinfo -- begin*/
4254                         x86_64_alu_imm_reg(cd, X86_64_SUB, 4 * 8, REG_SP);
4255                         x86_64_mov_reg_membase(cd, REG_ITMP2_XPC, REG_SP, 3 * 8);
4256                         x86_64_mov_imm_membase(cd,0,REG_SP,2*8);
4257                         x86_64_mov_imm_membase(cd,0,REG_SP,1*8);
4258                         x86_64_mov_imm_reg(cd,(ptrint)asm_prepare_native_stackinfo,REG_ITMP3);
4259                         x86_64_call_reg(cd,REG_ITMP3);
4260                         /*create stackinfo -- end*/
4261
4262
4263                         x86_64_mov_imm_reg(cd, (ptrint) new_nullpointerexception, REG_ITMP3);
4264                         x86_64_call_reg(cd, REG_ITMP3);
4265
4266                         /*remove stackinfo -- begin*/
4267                         x86_64_mov_imm_reg(cd,(ptrint)asm_remove_native_stackinfo,REG_ITMP3);
4268                         x86_64_call_reg(cd,REG_ITMP3);
4269                         /*remove stackinfo -- end*/
4270
4271                         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, REG_ITMP2_XPC);
4272                         x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
4273
4274                         x86_64_mov_imm_reg(cd, (ptrint) asm_handle_exception, REG_ITMP3);
4275                         x86_64_jmp_reg(cd, REG_ITMP3);
4276                 }
4277         }
4278
4279         /* generate code patching stub call code */
4280
4281         {
4282                 patchref    *pref;
4283                 codegendata *tmpcd;
4284                 ptrint       mcode;
4285
4286                 tmpcd = DNEW(codegendata);
4287
4288                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4289                         /* check size of code segment */
4290
4291                         MCODECHECK(2 * 8 + 128);
4292
4293                         /* Get machine code which is patched back in later. A             */
4294                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4295
4296                         xcodeptr = cd->mcodebase + pref->branchpos;
4297                         mcode = *((ptrint *) xcodeptr);
4298
4299 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4300                         /* create a virtual java_objectheader */
4301
4302                         *((ptrint *) (cd->mcodeptr + 0)) = 0;                    /* vftbl */
4303                         *((ptrint *) (cd->mcodeptr + 8)) = (ptrint) get_dummyLR(); /* monitorPtr */
4304
4305                         cd->mcodeptr += 16;
4306 #endif
4307
4308                         /* patch in `call rel32' to call the following code               */
4309
4310                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4311                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4312
4313                         /* move pointer to java_objectheader onto stack */
4314
4315 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4316                         x86_64_call_imm(cd, 0);
4317                         x86_64_alu_imm_membase(cd, X86_64_SUB, 5 + 2 * 8, REG_SP, 0);
4318 #else
4319                         x86_64_push_imm(cd, 0);
4320 #endif
4321
4322                         /* move machine code bytes and classinfo pointer into registers */
4323
4324                         x86_64_mov_imm_reg(cd, (ptrint) mcode, REG_ITMP3);
4325                         x86_64_push_reg(cd, REG_ITMP3);
4326                         x86_64_mov_imm_reg(cd, (ptrint) pref->ref, REG_ITMP3);
4327                         x86_64_push_reg(cd, REG_ITMP3);
4328
4329                         x86_64_mov_imm_reg(cd, (ptrint) pref->patcher, REG_ITMP3);
4330                         x86_64_push_reg(cd, REG_ITMP3);
4331
4332                         x86_64_mov_imm_reg(cd, (ptrint) asm_wrapper_patcher, REG_ITMP3);
4333                         x86_64_jmp_reg(cd, REG_ITMP3);
4334                 }
4335         }
4336         }
4337
4338         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4339 }
4340
4341
4342 /* function createcompilerstub *************************************************
4343
4344         creates a stub routine which calls the compiler
4345
4346 *******************************************************************************/
4347
4348 #define COMPSTUBSIZE    23
4349
4350 u1 *createcompilerstub(methodinfo *m)
4351 {
4352         u1 *s = CNEW(u1, COMPSTUBSIZE);     /* memory to hold the stub            */
4353         codegendata *cd;
4354         s4 dumpsize;
4355
4356         /* mark start of dump memory area */
4357
4358         dumpsize = dump_size();
4359
4360         cd = DNEW(codegendata);
4361         cd->mcodeptr = s;
4362
4363         /* code for the stub */
4364
4365         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1); /* pass method to compiler */
4366         x86_64_mov_imm_reg(cd, (ptrint) asm_call_jit_compiler, REG_ITMP3);
4367         x86_64_jmp_reg(cd, REG_ITMP3);
4368
4369 #if defined(STATISTICS)
4370         if (opt_stat)
4371                 count_cstub_len += COMPSTUBSIZE;
4372 #endif
4373
4374         /* release dump area */
4375
4376         dump_release(dumpsize);
4377
4378         return s;
4379 }
4380
4381
4382 /* function removecompilerstub *************************************************
4383
4384      deletes a compilerstub from memory  (simply by freeing it)
4385
4386 *******************************************************************************/
4387
4388 void removecompilerstub(u1 *stub)
4389 {
4390         CFREE(stub, COMPSTUBSIZE);
4391 }
4392
4393
4394 /* function: createnativestub **************************************************
4395
4396         creates a stub routine which calls a native method
4397
4398 *******************************************************************************/
4399
4400 /* #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
4401 /* static java_objectheader **(*callgetexceptionptrptr)() = builtin_get_exceptionptrptr; */
4402 /* #endif */
4403
4404 #define NATIVESTUBSIZE    1024           /* keep this size high enough!        */
4405 #define NATIVESTUB_DATA_SIZE (7*8)
4406
4407 u1 *createnativestub(functionptr f, methodinfo *m)
4408 {
4409         u1                 *s;              /* pointer to stub memory             */
4410         codegendata        *cd;
4411         registerdata       *rd;
4412         t_inlining_globals *id;
4413         s4                  dumpsize;
4414         s4                  stackframesize; /* size of stackframe if needed       */
4415         u1                 *tptr;
4416         s4                  iargs;          /* count of integer arguments         */
4417         s4                  fargs;          /* count of float arguments           */
4418         s4                  i;              /* counter                            */
4419         s4                  stubsize;
4420         u8                  *cs;
4421         bool                require_clinit_call;
4422
4423         void **callAddrPatchPos=0;
4424         u1 *jmpInstrPos=0;
4425         void **jmpInstrPatchPos=0;
4426
4427         /* initialize variables */
4428
4429         iargs = 0;
4430         fargs = 0;
4431
4432         /* mark start of dump memory area */
4433
4434         dumpsize = dump_size();
4435
4436         cd = DNEW(codegendata);
4437         rd = DNEW(registerdata);
4438         id = DNEW(t_inlining_globals);
4439
4440         /* setup registers before using it */
4441
4442         inlining_setup(m, id);
4443         reg_setup(m, rd, id);
4444
4445         /* set paramcount and paramtypes      */
4446
4447         method_descriptor2types(m);
4448
4449         /* count integer and float arguments */
4450
4451         tptr = m->paramtypes;
4452         for (i = 0; i < m->paramcount; i++) {
4453                 IS_INT_LNG_TYPE(*tptr++) ? iargs++ : fargs++;
4454         }
4455
4456         stubsize=NATIVESTUBSIZE;
4457         require_clinit_call= ((m->flags & ACC_STATIC) && !m->class->initialized);
4458         if (require_clinit_call) stubsize+=NATIVESTUB_DATA_SIZE;
4459         s = CNEW(u1, stubsize);       /* memory to hold the stub            */
4460
4461         if (require_clinit_call) {
4462                 cs = (u8*) (s+NATIVESTUB_DATA_SIZE);
4463                 *(cs - 7) = 0;                  /* extable size,padding               */
4464                 *(cs - 6) = 0;                  /* line number table start            */
4465                 *(cs - 5) = 0;                  /* line number table size             */
4466                 *(cs - 4) = 0;                  /* padding,fltsave                    */
4467                 *(cs - 3) = 0;                  /* intsave=0,isleaf=0                 */
4468                 *(cs - 2) = 0x0000000000000000;  /* frame size=0 (stack misalignment)  issync=0     */
4469 #if 0
4470                 *(cs - 2) = 0x0000000100000000;  /* frame size=1  issync=0     */
4471 #endif
4472                 *(cs - 1) = (u8) m;             /* method pointer                     */
4473         } else {
4474                 cs = (u8*) s;
4475         }
4476
4477         /* set some required varibles which are normally set by codegen_setup */
4478         cd->mcodebase = (u1*)cs;
4479         cd->mcodeptr = (u1*)cs;
4480         cd->patchrefs = NULL;
4481
4482         /* if function is static, check for initialized */
4483
4484         if (require_clinit_call) {
4485                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_clinit, m->class);
4486         }
4487
4488         if (JWNATIVEDEBUG(runverbose)) {
4489                 s4 l, s1;
4490
4491                 x86_64_alu_imm_reg(cd, X86_64_SUB, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4492
4493                 /* save integer and float argument registers */
4494
4495                 for (i = 0; i < INT_ARG_CNT; i++) {
4496                         x86_64_mov_reg_membase(cd, rd->argintregs[i], REG_SP, (1 + i) * 8);
4497                 }
4498
4499                 for (i = 0; i < FLT_ARG_CNT; i++) {
4500                         x86_64_movq_reg_membase(cd, rd->argfltregs[i], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4501                 }
4502
4503                 /* show integer hex code for float arguments */
4504
4505                 for (i = 0, l = 0; i < m->paramcount && i < INT_ARG_CNT; i++) {
4506                         /* if the paramtype is a float, we have to right shift all        */
4507                         /* following integer registers                                    */
4508
4509                         if (IS_FLT_DBL_TYPE(m->paramtypes[i])) {
4510                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--) {
4511                                         x86_64_mov_reg_reg(cd, rd->argintregs[s1], rd->argintregs[s1 + 1]);
4512                                 }
4513
4514                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[i]);
4515                                 l++;
4516                         }
4517                 }
4518
4519                 x86_64_mov_imm_reg(cd, (u8) m, REG_ITMP1);
4520                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
4521                 x86_64_mov_imm_reg(cd, (u8) builtin_trace_args, REG_ITMP1);
4522                 x86_64_call_reg(cd, REG_ITMP1);
4523
4524                 /* restore integer and float argument registers */
4525
4526                 for (i = 0; i < INT_ARG_CNT; i++) {
4527                         x86_64_mov_membase_reg(cd, REG_SP, (1 + i) * 8, rd->argintregs[i]);
4528                 }
4529
4530                 for (i = 0; i < FLT_ARG_CNT; i++) {
4531                         x86_64_movq_membase_reg(cd, REG_SP, (1 + INT_ARG_CNT + i) * 8, rd->argfltregs[i]);
4532                 }
4533
4534                 x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT + 1) * 8, REG_SP);
4535         }
4536
4537         /* 4 == additional size needed for native stack frame information*/
4538         x86_64_alu_imm_reg(cd, X86_64_SUB, (4+INT_ARG_CNT + FLT_ARG_CNT+1) * 8, REG_SP);
4539
4540         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, 0 * 8);
4541         x86_64_mov_reg_membase(cd, rd->argintregs[1], REG_SP, 1 * 8);
4542         x86_64_mov_reg_membase(cd, rd->argintregs[2], REG_SP, 2 * 8);
4543         x86_64_mov_reg_membase(cd, rd->argintregs[3], REG_SP, 3 * 8);
4544         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 4 * 8);
4545         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 5 * 8);
4546
4547         x86_64_movq_reg_membase(cd, rd->argfltregs[0], REG_SP, 6 * 8);
4548         x86_64_movq_reg_membase(cd, rd->argfltregs[1], REG_SP, 7 * 8);
4549         x86_64_movq_reg_membase(cd, rd->argfltregs[2], REG_SP, 8 * 8);
4550         x86_64_movq_reg_membase(cd, rd->argfltregs[3], REG_SP, 9 * 8);
4551         x86_64_movq_reg_membase(cd, rd->argfltregs[4], REG_SP, 10 * 8);
4552         x86_64_movq_reg_membase(cd, rd->argfltregs[5], REG_SP, 11 * 8);
4553         x86_64_movq_reg_membase(cd, rd->argfltregs[6], REG_SP, 12 * 8);
4554         x86_64_movq_reg_membase(cd, rd->argfltregs[7], REG_SP, 13 * 8);
4555
4556 /*
4557 15+
4558 0*8        void *oldThreadspecificHeadValue;
4559 1*8        void **addressOfThreadspecificHead;
4560 2*8        methodinfo *method;
4561 3*8        void *beginOfJavaStackframe; only used if != 0
4562 4*8        void *returnToFromNative;
4563 */
4564
4565         /* CREATE DYNAMIC STACK INFO -- BEGIN   offsets:15,16,17,18*/
4566         x86_64_mov_imm_membase(cd, 0, REG_SP, 18*8);
4567         x86_64_mov_imm_membase(cd, (ptrint)m, REG_SP, 17*8);
4568         /*x86_64_mov_imm_membase(cd, 0, REG_SP, 17*8);*/
4569
4570         x86_64_mov_imm_reg(cd, (u8) builtin_asm_get_stackframeinfo,REG_ITMP1);
4571         x86_64_call_reg(cd,REG_ITMP1);
4572
4573         x86_64_mov_reg_membase(cd,REG_RESULT,REG_SP,16*8);
4574         x86_64_mov_membase_reg(cd,REG_RESULT,0,REG_ITMP2);
4575         x86_64_mov_reg_membase(cd,REG_ITMP2,REG_SP,15*8);
4576         x86_64_mov_reg_reg(cd,REG_SP,REG_ITMP2);
4577         x86_64_alu_imm_reg(cd, X86_64_ADD, (1+INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_ITMP2);
4578         x86_64_mov_reg_membase(cd,REG_ITMP2,REG_RESULT,0);
4579
4580 #if 0
4581
4582         i386_mov_imm_membase(cd,0,REG_SP,stackframesize-4);
4583         i386_mov_imm_membase(cd, (s4) m, REG_SP,stackframesize-8);
4584         i386_mov_imm_reg(cd, (s4) builtin_asm_get_stackframeinfo, REG_ITMP1);
4585         i386_call_reg(cd, REG_ITMP1);
4586         i386_mov_reg_membase(cd, REG_RESULT,REG_SP,stackframesize-12); /*save thread specific pointer*/
4587         i386_mov_membase_reg(cd, REG_RESULT,0,REG_ITMP2);
4588         i386_mov_reg_membase(cd, REG_ITMP2,REG_SP,stackframesize-16); /*save previous value of memory adress pointed to by thread specific pointer*/
4589         i386_mov_reg_reg(cd, REG_SP,REG_ITMP2);
4590         i386_alu_imm_reg(cd, I386_ADD,stackframesize-16,REG_ITMP2);
4591         i386_mov_reg_membase(cd, REG_ITMP2,REG_RESULT,0);
4592 #endif
4593         /* CREATE DYNAMIC STACK INFO -- END*/
4594
4595         STATS({
4596                 x86_64_mov_imm_reg(cd,(u8)nativeinvokation,REG_ITMP1);
4597                 x86_64_call_reg(cd,REG_ITMP1);
4598         })
4599
4600 #if !defined(STATIC_CLASSPATH)
4601         /* call method to resolve native function if needed */
4602         if (f == NULL) {
4603                 /* needed to patch a jump over this block */
4604                 x86_64_jmp_imm(cd, 0);
4605                 jmpInstrPos = cd->mcodeptr - 4;
4606
4607
4608                 x86_64_mov_imm_reg(cd, (ptrint) m, rd->argintregs[0]);
4609
4610                 x86_64_mov_imm_reg(cd, 0, rd->argintregs[1]);
4611                 callAddrPatchPos = cd->mcodeptr - 8; /* at this position the place is specified where the native function adress should be patched into*/
4612
4613                 x86_64_mov_imm_reg(cd, 0, rd->argintregs[2]);
4614                 jmpInstrPatchPos = cd->mcodeptr - 8;
4615
4616                 x86_64_mov_imm_reg(cd, (ptrint) jmpInstrPos, rd->argintregs[3]);
4617
4618                 x86_64_mov_imm_reg(cd, (ptrint) codegen_resolve_native, REG_ITMP1);
4619                 x86_64_call_reg(cd, REG_ITMP1);
4620
4621                 *(jmpInstrPatchPos) = cd->mcodeptr - jmpInstrPos - 1-3; /*=opcode jmp_imm size*/
4622
4623         }
4624
4625 #endif
4626
4627
4628         x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, rd->argintregs[0]);
4629         x86_64_mov_membase_reg(cd, REG_SP, 1 * 8, rd->argintregs[1]);
4630         x86_64_mov_membase_reg(cd, REG_SP, 2 * 8, rd->argintregs[2]);
4631         x86_64_mov_membase_reg(cd, REG_SP, 3 * 8, rd->argintregs[3]);
4632         x86_64_mov_membase_reg(cd, REG_SP, 4 * 8, rd->argintregs[4]);
4633         x86_64_mov_membase_reg(cd, REG_SP, 5 * 8, rd->argintregs[5]);
4634
4635         x86_64_movq_membase_reg(cd, REG_SP, 6 * 8, rd->argfltregs[0]);
4636         x86_64_movq_membase_reg(cd, REG_SP, 7 * 8, rd->argfltregs[1]);
4637         x86_64_movq_membase_reg(cd, REG_SP, 8 * 8, rd->argfltregs[2]);
4638         x86_64_movq_membase_reg(cd, REG_SP, 9 * 8, rd->argfltregs[3]);
4639         x86_64_movq_membase_reg(cd, REG_SP, 10 * 8, rd->argfltregs[4]);
4640         x86_64_movq_membase_reg(cd, REG_SP, 11 * 8, rd->argfltregs[5]);
4641         x86_64_movq_membase_reg(cd, REG_SP, 12 * 8, rd->argfltregs[6]);
4642         x86_64_movq_membase_reg(cd, REG_SP, 13 * 8, rd->argfltregs[7]);
4643
4644         x86_64_alu_imm_reg(cd, X86_64_ADD, (INT_ARG_CNT + FLT_ARG_CNT+1) * 8, REG_SP);
4645
4646         /* save argument registers on stack -- if we have to */
4647
4648         if ((((m->flags & ACC_STATIC) && iargs > (INT_ARG_CNT - 2)) || iargs > (INT_ARG_CNT - 1)) ||
4649                 (fargs > FLT_ARG_CNT)) {
4650                 s4 paramshiftcnt;
4651                 s4 stackparamcnt;
4652
4653                 paramshiftcnt = 0;
4654                 stackparamcnt = 0;
4655
4656                 /* do we need to shift integer argument register onto stack? */
4657
4658                 if ((m->flags & ACC_STATIC) && iargs > (INT_ARG_CNT - 2)) {
4659                         /* do we need to shift 2 arguments? */
4660                         if (iargs > (INT_ARG_CNT - 1)) {
4661                                 paramshiftcnt = 2;
4662
4663                         } else {
4664                                 paramshiftcnt = 1;
4665                         }
4666
4667                 } else if (iargs > (INT_ARG_CNT - 1)) {
4668                         paramshiftcnt = 1;
4669                 }
4670
4671                 /* calculate required stack space */
4672
4673                 stackparamcnt += (iargs > INT_ARG_CNT) ? iargs - INT_ARG_CNT : 0;
4674                 stackparamcnt += (fargs > FLT_ARG_CNT) ? fargs - FLT_ARG_CNT : 0;
4675
4676
4677                 stackframesize = stackparamcnt + paramshiftcnt;
4678
4679                 /* keep stack 16-byte aligned */
4680                 if (!(stackframesize & 0x1))
4681                         stackframesize++;
4682
4683                 x86_64_alu_imm_reg(cd, X86_64_SUB, stackframesize * 8, REG_SP);
4684
4685                 /* shift integer arguments if required */
4686
4687                 if ((m->flags & ACC_STATIC) && iargs > (INT_ARG_CNT - 2)) {
4688                         /* do we need to shift 2 arguments? */
4689                         if (iargs > (INT_ARG_CNT - 1))
4690                                 x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 1 * 8);
4691
4692                         x86_64_mov_reg_membase(cd, rd->argintregs[4], REG_SP, 0 * 8);
4693
4694                 } else if (iargs > (INT_ARG_CNT - 1)) {
4695                         x86_64_mov_reg_membase(cd, rd->argintregs[5], REG_SP, 0 * 8);
4696                 }
4697
4698                 /* copy stack arguments into new stack frame -- if any */
4699                 for (i = 0; i < stackparamcnt; i++) {
4700                         x86_64_mov_membase_reg(cd, REG_SP, (stackframesize + 1 + i+4) * 8, REG_ITMP1);  /* 4==additional size for stackrace data*/
4701                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, (paramshiftcnt + i) * 8);
4702                 }
4703
4704         } else {
4705                 /* keep stack 16-byte aligned */
4706                 x86_64_alu_imm_reg(cd, X86_64_SUB, 1 * 8, REG_SP);
4707                 stackframesize = 1;
4708         }
4709
4710         /* shift integer arguments for `env' and `class' arguments */
4711
4712         if (m->flags & ACC_STATIC) {
4713                 /* shift iargs count if less than INT_ARG_CNT, or all */
4714                 for (i = (iargs < (INT_ARG_CNT - 2)) ? iargs : (INT_ARG_CNT - 2); i >= 0; i--) {
4715                         x86_64_mov_reg_reg(cd, rd->argintregs[i], rd->argintregs[i + 2]);
4716                 }
4717
4718                 /* put class into second argument register */
4719                 x86_64_mov_imm_reg(cd, (u8) m->class, rd->argintregs[1]);
4720
4721         } else {
4722                 /* shift iargs count if less than INT_ARG_CNT, or all */
4723                 for (i = (iargs < (INT_ARG_CNT - 1)) ? iargs : (INT_ARG_CNT - 1); i >= 0; i--) {
4724                         x86_64_mov_reg_reg(cd, rd->argintregs[i], rd->argintregs[i + 1]);
4725                 }
4726         }
4727
4728         /* put env into first argument register */
4729         x86_64_mov_imm_reg(cd, (u8) &env, rd->argintregs[0]);
4730
4731         /* do the native function call */
4732         x86_64_mov_imm_reg(cd, (u8) f, REG_ITMP1);
4733 #if !defined(STATIC_CLASSPATH)
4734         if (f == NULL)
4735                 (*callAddrPatchPos) = cd->mcodeptr - 8;
4736 #endif
4737         x86_64_call_reg(cd, REG_ITMP1);
4738
4739         /* remove stackframe if there is one */
4740         if (stackframesize) {
4741                 x86_64_alu_imm_reg(cd, X86_64_ADD, stackframesize * 8, REG_SP);
4742         }
4743
4744         /*REMOVE DYNAMIC STACK INFO -BEGIN */
4745         x86_64_mov_reg_membase(cd,REG_RESULT,REG_SP,2*8);
4746         x86_64_mov_membase_reg(cd,REG_SP,0*8,REG_ITMP2);
4747         x86_64_mov_membase_reg(cd,REG_SP,1*8,REG_RESULT);
4748         x86_64_mov_reg_membase(cd,REG_ITMP2,REG_RESULT,0);
4749         x86_64_mov_membase_reg(cd,REG_SP,2*8,REG_RESULT);
4750 #if 0
4751         i386_push_reg(cd, REG_RESULT2);
4752         i386_mov_membase_reg(cd, REG_SP,stackframesize-12,REG_ITMP2); /*old value*/
4753         i386_mov_membase_reg(cd, REG_SP,stackframesize-8,REG_RESULT2); /*pointer*/
4754         i386_mov_reg_membase(cd, REG_ITMP2,REG_RESULT2,0);
4755         i386_pop_reg(cd, REG_RESULT2);
4756 #endif
4757         /*REMOVE DYNAMIC STACK INFO -END */
4758
4759         x86_64_alu_imm_reg(cd, X86_64_ADD, 4 * 8, REG_SP);
4760
4761         if (JWNATIVEDEBUG(runverbose)) {
4762                 x86_64_alu_imm_reg(cd, X86_64_SUB, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
4763
4764                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
4765                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
4766
4767                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
4768                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
4769                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4770                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4771
4772                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
4773                 x86_64_call_reg(cd, REG_ITMP1);
4774
4775                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
4776                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
4777
4778                 x86_64_alu_imm_reg(cd, X86_64_ADD, 3 * 8, REG_SP);    /* keep stack 16-byte aligned */
4779         }
4780
4781         /* check for exception */
4782
4783 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4784         x86_64_push_reg(cd, REG_RESULT);
4785 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4786         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4787         x86_64_call_reg(cd, REG_ITMP3);
4788         x86_64_mov_membase_reg(cd, REG_RESULT, 0, REG_ITMP3);
4789         x86_64_pop_reg(cd, REG_RESULT);
4790 #else
4791         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
4792         x86_64_mov_membase_reg(cd, REG_ITMP3, 0, REG_ITMP3);
4793 #endif
4794         x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
4795         x86_64_jcc(cd, X86_64_CC_NE, 1);
4796
4797         x86_64_ret(cd);
4798
4799         /* handle exception */
4800
4801 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4802         x86_64_push_reg(cd, REG_ITMP3);
4803 /*      x86_64_call_mem(cd, (u8) &callgetexceptionptrptr); */
4804         x86_64_mov_imm_reg(cd, (u8) builtin_get_exceptionptrptr, REG_ITMP3);
4805         x86_64_call_reg(cd, REG_ITMP3);
4806         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4807         x86_64_pop_reg(cd, REG_ITMP1_XPTR);
4808 #else
4809         x86_64_mov_reg_reg(cd, REG_ITMP3, REG_ITMP1_XPTR);
4810         x86_64_mov_imm_reg(cd, (u8) &_exceptionptr, REG_ITMP3);
4811         x86_64_alu_reg_reg(cd, X86_64_XOR, REG_ITMP2, REG_ITMP2);
4812         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_ITMP3, 0);    /* clear exception pointer */
4813 #endif
4814
4815         x86_64_mov_membase_reg(cd, REG_SP, 0, REG_ITMP2_XPC);    /* get return address from stack */
4816         x86_64_alu_imm_reg(cd, X86_64_SUB, 3, REG_ITMP2_XPC);    /* callq */
4817
4818         x86_64_mov_imm_reg(cd, (u8) asm_handle_nat_exception, REG_ITMP3);
4819         x86_64_jmp_reg(cd, REG_ITMP3);
4820
4821
4822         /* patch in a <clinit> call if required ***********************************/
4823
4824         {
4825                 u1          *xcodeptr;
4826                 patchref    *pref;
4827                 codegendata *tmpcd;
4828                 ptrint       mcode;
4829
4830                 tmpcd = DNEW(codegendata);
4831
4832                 /* there can only be one patch ref entry                              */
4833                 pref = cd->patchrefs;
4834
4835                 if (pref) {
4836                         /* Get machine code which is patched back in later. A             */
4837                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4838
4839                         xcodeptr = cd->mcodebase + pref->branchpos;
4840                         mcode = *((ptrint *) xcodeptr);
4841
4842 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4843                         /* create a virtual java_objectheader */
4844
4845                         *((ptrint *) (cd->mcodeptr + 0)) = 0;                    /* vftbl */
4846                         *((ptrint *) (cd->mcodeptr + 8)) = (ptrint) get_dummyLR(); /* monitorPtr */
4847
4848                         cd->mcodeptr += 16;
4849 #endif
4850
4851                         /* patch in `call rel32' to call the following code               */
4852
4853                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4854                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4855
4856                         /* move pointer to java_objectheader onto stack */
4857
4858 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4859                         x86_64_call_imm(cd, 0);
4860                         x86_64_alu_imm_membase(cd, X86_64_SUB, 5 + 2 * 8, REG_SP, 0);
4861 #else
4862                         x86_64_push_imm(cd, 0);
4863 #endif
4864
4865                         /* move machine code bytes and classinfo pointer into registers */
4866
4867                         x86_64_mov_imm_reg(cd, (ptrint) mcode, REG_ITMP3);
4868                         x86_64_push_reg(cd, REG_ITMP3);
4869                         x86_64_mov_imm_reg(cd, (ptrint) pref->ref, REG_ITMP3);
4870                         x86_64_push_reg(cd, REG_ITMP3);
4871
4872                         x86_64_mov_imm_reg(cd, (ptrint) pref->patcher, REG_ITMP3);
4873                         x86_64_push_reg(cd, REG_ITMP3);
4874
4875                         x86_64_mov_imm_reg(cd, (ptrint) asm_wrapper_patcher, REG_ITMP3);
4876                         x86_64_jmp_reg(cd, REG_ITMP3);
4877
4878                         codegen_insertmethod((functionptr) cs, (functionptr) cd->mcodeptr);
4879                         /*printf("codegen_insertmethod (nativestub) %s.%s:   %p  %p\n",
4880                                                                 m->class->name->text,m->name->text,cs,cd->mcodeptr);*/
4881                 }
4882         }
4883
4884         /*printf("(nativestub) %s.%s:  %p  %p\n",m->class->name->text,m->name->text,cs,cd->mcodeptr);*/
4885
4886         /* Check if the stub size is big enough to hold the whole stub generated. */
4887         /* If not, this can lead into unpredictable crashes, because of heap      */
4888         /* corruption.                                                            */
4889         if ((s4) (cd->mcodeptr - s) > stubsize) {
4890                 throw_cacao_exception_exit(string_java_lang_InternalError,
4891                                                                    "Native stub size %d is to small for current stub size %d",
4892                                                                    stubsize, (s4) (cd->mcodeptr - s));
4893         }
4894
4895
4896 #if defined(STATISTICS)
4897         if (opt_stat)
4898                 count_nstub_len += stubsize;
4899 #endif
4900
4901         /* release dump area */
4902
4903         dump_release(dumpsize);
4904
4905         return (u1*) cs;
4906 }
4907
4908
4909 /* function: removenativestub **************************************************
4910
4911     removes a previously created native-stub from memory
4912
4913 *******************************************************************************/
4914
4915 void removenativestub(u1 *stub)
4916 {
4917         CFREE(stub, NATIVESTUBSIZE);
4918 }
4919
4920
4921 /*
4922  * These are local overrides for various environment variables in Emacs.
4923  * Please do not remove this and leave it at the end of the file, where
4924  * Emacs will automagically detect them.
4925  * ---------------------------------------------------------------------
4926  * Local variables:
4927  * mode: c
4928  * indent-tabs-mode: t
4929  * c-basic-offset: 4
4930  * tab-width: 4
4931  * End:
4932  */