src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31
  32    $Id: codegen.c 3661 2005-11-11 14:02:45Z twisti $
  33
  34 */
  35
  36
  37 #include <stdio.h>
  38
  39 #include "config.h"
  40 #include "vm/types.h"
  41
  42 #include "md.h"
  43 #include "md-abi.h"
  44 #include "md-abi.inc"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emitfuncs.h"
  49
  50 #include "cacao/cacao.h"
  51 #include "native/native.h"
  52 #include "vm/global.h"
  53 #include "vm/builtin.h"
  54 #include "vm/loader.h"
  55 #include "vm/statistics.h"
  56 #include "vm/stringlocal.h"
  57 #include "vm/tables.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/codegen.inc"
  60 #include "vm/jit/jit.h"
  61
  62 #if defined(LSRA)
  63 # include "vm/jit/lsra.inc"
  64 #endif
  65
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.h"
  68 #include "vm/jit/patcher.h"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/reg.inc"
  71
  72
  73 /* codegen *********************************************************************
  74
  75    Generates machine code.
  76
  77 *******************************************************************************/
  78
  79 bool codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  80 {
  81         s4                  len, s1, s2, s3, d, disp;
  82         u2                  currentline;
  83         ptrint              a;
  84         s4                  parentargs_base;
  85         stackptr            src;
  86         varinfo            *var;
  87         basicblock         *bptr;
  88         instruction        *iptr;
  89         exceptiontable     *ex;
  90         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  91         builtintable_entry *bte;
  92         methoddesc         *md;
  93
  94         /* prevent compiler warnings */
  95
  96         d = 0;
  97         lm = NULL;
  98         bte = NULL;
  99
 100         {
 101         s4 i, p, t, l;
 102         s4 savedregs_num;
 103
 104         savedregs_num = 0;
 105
 106         /* space to save used callee saved registers */
 107
 108         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 109         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 110
 111         parentargs_base = rd->memuse + savedregs_num;
 112
 113 #if defined(USE_THREADS)
 114         /* space to save argument of monitor_enter */
 115
 116         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 117                 parentargs_base++;
 118 #endif
 119
 120     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 121         /* code e.g. libc or jni (alignment problems with movaps).                */
 122
 123         if (!m->isleafmethod || runverbose)
 124                 parentargs_base |= 0x1;
 125
 126         /* create method header */
 127
 128         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 129         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 130
 131 #if defined(USE_THREADS)
 132         /* IsSync contains the offset relative to the stack pointer for the
 133            argument of monitor_exit used in the exception handler. Since the
 134            offset could be zero and give a wrong meaning of the flag it is
 135            offset by one.
 136         */
 137
 138         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 139                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 140         else
 141 #endif
 142                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 143
 144         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 145         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 146         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 147
 148         (void) dseg_addlinenumbertablesize(cd);
 149
 150         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 151
 152         /* create exception table */
 153
 154         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 155                 dseg_addtarget(cd, ex->start);
 156                 dseg_addtarget(cd, ex->end);
 157                 dseg_addtarget(cd, ex->handler);
 158                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 159         }
 160
 161         /* initialize mcode variables */
 162
 163         cd->mcodeptr = (u1 *) cd->mcodebase;
 164         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 165         MCODECHECK(512);
 166
 167         /* initialize the last patcher pointer */
 168
 169         cd->lastmcodeptr = cd->mcodeptr;
 170
 171         /* create stack frame (if necessary) */
 172
 173         if (parentargs_base)
 174                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 175
 176         /* save used callee saved registers */
 177
 178         p = parentargs_base;
 179         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 180                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 181         }
 182         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 183                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 184         }
 185
 186         /* take arguments out of register or stack frame */
 187
 188         md = m->parseddesc;
 189
 190         for (p = 0, l = 0; p < md->paramcount; p++) {
 191                 t = md->paramtypes[p].type;
 192                 var = &(rd->locals[l][t]);
 193                 l++;
 194                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 195                         l++;
 196                 if (var->type < 0)
 197                         continue;
 198                 s1 = md->params[p].regoff;
 199                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 200                         s2 = rd->argintregs[s1];
 201                         if (!md->params[p].inmemory) {           /* register arguments    */
 202                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 203                                         M_INTMOVE(s2, var->regoff);
 204
 205                                 } else {                             /* reg arg -> spilled    */
 206                                     M_LST(s2, REG_SP, var->regoff * 8);
 207                                 }
 208
 209                         } else {                                 /* stack arguments       */
 210                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 211                                         /* + 8 for return address */
 212                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 213
 214                                 } else {                             /* stack arg -> spilled  */
 215                                         var->regoff = parentargs_base + s1 + 1;
 216                                 }
 217                         }
 218
 219                 } else {                                     /* floating args         */
 220                         if (!md->params[p].inmemory) {           /* register arguments    */
 221                                 s2 = rd->argfltregs[s1];
 222                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 223                                         M_FLTMOVE(s2, var->regoff);
 224
 225                                 } else {                                         /* reg arg -> spilled    */
 226                                         M_DST(s2, REG_SP, var->regoff * 8);
 227                                 }
 228
 229                         } else {                                 /* stack arguments       */
 230                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 231                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 232
 233                                 } else {
 234                                         var->regoff = parentargs_base + s1 + 1;
 235                                 }
 236                         }
 237                 }
 238         }  /* end for */
 239
 240         /* save monitorenter argument */
 241
 242 #if defined(USE_THREADS)
 243         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 244                 /* stack offset for monitor argument */
 245
 246                 s1 = rd->memuse;
 247
 248                 if (runverbose) {
 249                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 250
 251                         for (p = 0; p < INT_ARG_CNT; p++)
 252                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 253
 254                         for (p = 0; p < FLT_ARG_CNT; p++)
 255                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 256
 257                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 258                 }
 259
 260                 /* decide which monitor enter function to call */
 261
 262                 if (m->flags & ACC_STATIC) {
 263                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 264                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 265                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 266                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 267                         x86_64_call_reg(cd, REG_ITMP1);
 268
 269                 } else {
 270                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 271                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 272                         codegen_addxnullrefs(cd, cd->mcodeptr);
 273                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 274                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 275                         x86_64_call_reg(cd, REG_ITMP1);
 276                 }
 277
 278                 if (runverbose) {
 279                         for (p = 0; p < INT_ARG_CNT; p++)
 280                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 281
 282                         for (p = 0; p < FLT_ARG_CNT; p++)
 283                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 284
 285                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 286                 }
 287         }
 288 #endif
 289
 290         /* Copy argument registers to stack and call trace function with pointer  */
 291         /* to arguments on stack.                                                 */
 292
 293         if (runverbose || opt_stat) {
 294                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 295
 296                 /* save integer argument registers */
 297
 298                 for (p = 0; p < INT_ARG_CNT; p++)
 299                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 300
 301                 /* save float argument registers */
 302
 303                 for (p = 0; p < FLT_ARG_CNT; p++)
 304                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 305
 306                 /* save temporary registers for leaf methods */
 307
 308                 if (m->isleafmethod) {
 309                         for (p = 0; p < INT_TMP_CNT; p++)
 310                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 311
 312                         for (p = 0; p < FLT_TMP_CNT; p++)
 313                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 314                 }
 315
 316                 if (runverbose) {
 317                         /* show integer hex code for float arguments */
 318
 319                         for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 320                                 /* if the paramtype is a float, we have to right shift all    */
 321                                 /* following integer registers                                */
 322
 323                                 if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 324                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 325                                                 M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 326                                         }
 327
 328                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 329                                         l++;
 330                                 }
 331                         }
 332
 333                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 334                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 335                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 336                         x86_64_call_reg(cd, REG_ITMP1);
 337                 }
 338                 STATS({
 339                         x86_64_mov_imm_reg(cd, (ptrint) compiledinvokation, REG_ITMP1);
 340                         x86_64_call_reg(cd, REG_ITMP1);
 341                 })
 342
 343                 /* restore integer argument registers */
 344
 345                 for (p = 0; p < INT_ARG_CNT; p++)
 346                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 347
 348                 /* restore float argument registers */
 349
 350                 for (p = 0; p < FLT_ARG_CNT; p++)
 351                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 352
 353                 /* restore temporary registers for leaf methods */
 354
 355                 if (m->isleafmethod) {
 356                         for (p = 0; p < INT_TMP_CNT; p++)
 357                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 358
 359                         for (p = 0; p < FLT_TMP_CNT; p++)
 360                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 361                 }
 362
 363                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 364         }
 365
 366         }
 367
 368         /* end of header generation */
 369
 370         /* walk through all basic blocks */
 371         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 372
 373                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 374
 375                 if (bptr->flags >= BBREACHED) {
 376
 377                         /* branch resolving */
 378
 379                         branchref *bref;
 380                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 381                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 382                                                                   bref->branchpos,
 383                                                                   bptr->mpc);
 384                         }
 385
 386                 /* copy interface registers to their destination */
 387
 388                 src = bptr->instack;
 389                 len = bptr->indepth;
 390                 MCODECHECK(512);
 391
 392 #ifdef LSRA
 393                 if (opt_lsra) {
 394                         while (src != NULL) {
 395                                 len--;
 396                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 397                                         if (bptr->type == BBTYPE_SBR) {
 398                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 399                                                 if (!(src->flags & INMEMORY))
 400                                                         d= src->regoff;
 401                                                 else
 402                                                         d=REG_ITMP1;
 403                                                 x86_64_pop_reg(cd, d);
 404                                                 store_reg_to_var_int(src, d);
 405
 406                                         } else if (bptr->type == BBTYPE_EXH) {
 407                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 408                                                 if (!(src->flags & INMEMORY))
 409                                                         d= src->regoff;
 410                                                 else
 411                                                         d=REG_ITMP1;
 412                                                 M_INTMOVE(REG_ITMP1, d);
 413                                                 store_reg_to_var_int(src, d);
 414                                         }
 415                                 }
 416                                 src = src->prev;
 417                         }
 418
 419                 } else {
 420 #endif
 421
 422         while (src != NULL) {
 423                         len--;
 424                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 425                                 if (bptr->type == BBTYPE_SBR) {
 426                                         d = reg_of_var(rd, src, REG_ITMP1);
 427                                         x86_64_pop_reg(cd, d);
 428                                         store_reg_to_var_int(src, d);
 429
 430                                 } else if (bptr->type == BBTYPE_EXH) {
 431                                         d = reg_of_var(rd, src, REG_ITMP1);
 432                                         M_INTMOVE(REG_ITMP1, d);
 433                                         store_reg_to_var_int(src, d);
 434                                 }
 435
 436                         } else {
 437                                 d = reg_of_var(rd, src, REG_ITMP1);
 438                                 if ((src->varkind != STACKVAR)) {
 439                                         s2 = src->type;
 440                                         if (IS_FLT_DBL_TYPE(s2)) {
 441                                                 s1 = rd->interfaces[len][s2].regoff;
 442                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 443                                                         M_FLTMOVE(s1, d);
 444
 445                                                 } else {
 446                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 447                                                 }
 448                                                 store_reg_to_var_flt(src, d);
 449
 450                                         } else {
 451                                                 s1 = rd->interfaces[len][s2].regoff;
 452                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 453                                                         M_INTMOVE(s1, d);
 454
 455                                                 } else {
 456                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 457                                                 }
 458                                                 store_reg_to_var_int(src, d);
 459                                         }
 460                                 }
 461                         }
 462                         src = src->prev;
 463                 }
 464 #ifdef LSRA
 465                 }
 466 #endif
 467                 /* walk through all instructions */
 468
 469                 src = bptr->instack;
 470                 len = bptr->icount;
 471                 currentline=0;
 472
 473                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 474                         if (iptr->line != currentline) {
 475                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 476                                 currentline = iptr->line;
 477                         }
 478
 479                         MCODECHECK(1024);                         /* 1KB should be enough */
 480
 481                         switch (iptr->opc) {
 482                         case ICMD_INLINE_START: /* internal ICMDs                         */
 483                         case ICMD_INLINE_END:
 484                                 break;
 485
 486                         case ICMD_NOP:    /* ...  ==> ...                                 */
 487                                 break;
 488
 489                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 490                                 if (src->flags & INMEMORY) {
 491                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 492
 493                                 } else {
 494                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 495                                 }
 496                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 497                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 498                                 break;
 499
 500                 /* constant operations ************************************************/
 501
 502                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 503                                       /* op1 = 0, val.i = constant                    */
 504
 505                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 506                         if (iptr->val.i == 0) {
 507                                 M_XOR(d, d);
 508                         } else {
 509                                 M_IMOV_IMM(iptr->val.i, d);
 510                         }
 511                         store_reg_to_var_int(iptr->dst, d);
 512                         break;
 513
 514                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 515                                       /* op1 = 0, val.l = constant                    */
 516
 517                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 518                         if (iptr->val.l == 0) {
 519                                 M_XOR(d, d);
 520                         } else {
 521                                 M_MOV_IMM(iptr->val.l, d);
 522                         }
 523                         store_reg_to_var_int(iptr->dst, d);
 524                         break;
 525
 526                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 527                                       /* op1 = 0, val.f = constant                    */
 528
 529                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 530                         disp = dseg_addfloat(cd, iptr->val.f);
 531                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 532                         store_reg_to_var_flt(iptr->dst, d);
 533                         break;
 534
 535                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 536                                       /* op1 = 0, val.d = constant                    */
 537
 538                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 539                         disp = dseg_adddouble(cd, iptr->val.d);
 540                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 541                         store_reg_to_var_flt(iptr->dst, d);
 542                         break;
 543
 544                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 545                                       /* op1 = 0, val.a = constant                    */
 546
 547                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 548
 549                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 550                                 codegen_addpatchref(cd, cd->mcodeptr,
 551                                                                         PATCHER_aconst,
 552                                                                         (unresolved_class *) iptr->target, 0);
 553
 554                                 if (opt_showdisassemble) {
 555                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 556                                 }
 557
 558                                 M_MOV_IMM((ptrint) iptr->val.a, d);
 559
 560                         } else {
 561                                 if (iptr->val.a == 0) {
 562                                         M_XOR(d, d);
 563                                 } else {
 564                                         M_MOV_IMM((ptrint) iptr->val.a, d);
 565                                 }
 566                         }
 567                         store_reg_to_var_int(iptr->dst, d);
 568                         break;
 569
 570
 571                 /* load/store operations **********************************************/
 572
 573                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 574                                       /* op1 = local variable                         */
 575
 576                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 577                         if ((iptr->dst->varkind == LOCALVAR) &&
 578                             (iptr->dst->varnum == iptr->op1)) {
 579                                 break;
 580                         }
 581                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 582                         if (var->flags & INMEMORY) {
 583                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 584                                 store_reg_to_var_int(iptr->dst, d);
 585
 586                         } else {
 587                                 if (iptr->dst->flags & INMEMORY) {
 588                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 589
 590                                 } else {
 591                                         M_INTMOVE(var->regoff, d);
 592                                 }
 593                         }
 594                         break;
 595
 596                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 597                 case ICMD_ALOAD:      /* op1 = local variable                         */
 598
 599                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 600                         if ((iptr->dst->varkind == LOCALVAR) &&
 601                             (iptr->dst->varnum == iptr->op1)) {
 602                                 break;
 603                         }
 604                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 605                         if (var->flags & INMEMORY) {
 606                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 607                                 store_reg_to_var_int(iptr->dst, d);
 608
 609                         } else {
 610                                 if (iptr->dst->flags & INMEMORY) {
 611                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 612
 613                                 } else {
 614                                         M_INTMOVE(var->regoff, d);
 615                                 }
 616                         }
 617                         break;
 618
 619                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 620                 case ICMD_DLOAD:      /* op1 = local variable                         */
 621
 622                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 623                         if ((iptr->dst->varkind == LOCALVAR) &&
 624                             (iptr->dst->varnum == iptr->op1)) {
 625                                 break;
 626                         }
 627                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 628                         if (var->flags & INMEMORY) {
 629                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 630                                 store_reg_to_var_flt(iptr->dst, d);
 631
 632                         } else {
 633                                 if (iptr->dst->flags & INMEMORY) {
 634                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 635
 636                                 } else {
 637                                         M_FLTMOVE(var->regoff, d);
 638                                 }
 639                         }
 640                         break;
 641
 642                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 643                 case ICMD_LSTORE:     /* op1 = local variable                         */
 644                 case ICMD_ASTORE:
 645
 646                         if ((src->varkind == LOCALVAR) &&
 647                             (src->varnum == iptr->op1)) {
 648                                 break;
 649                         }
 650                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 651                         if (var->flags & INMEMORY) {
 652                                 var_to_reg_int(s1, src, REG_ITMP1);
 653                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 654
 655                         } else {
 656                                 var_to_reg_int(s1, src, var->regoff);
 657                                 M_INTMOVE(s1, var->regoff);
 658                         }
 659                         break;
 660
 661                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 662                 case ICMD_DSTORE:     /* op1 = local variable                         */
 663
 664                         if ((src->varkind == LOCALVAR) &&
 665                             (src->varnum == iptr->op1)) {
 666                                 break;
 667                         }
 668                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 669                         if (var->flags & INMEMORY) {
 670                                 var_to_reg_flt(s1, src, REG_FTMP1);
 671                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 672
 673                         } else {
 674                                 var_to_reg_flt(s1, src, var->regoff);
 675                                 M_FLTMOVE(s1, var->regoff);
 676                         }
 677                         break;
 678
 679
 680                 /* pop/dup/swap operations ********************************************/
 681
 682                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 683
 684                 case ICMD_POP:        /* ..., value  ==> ...                          */
 685                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 686                         break;
 687
 688                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 689                         M_COPY(src, iptr->dst);
 690                         break;
 691
 692                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 693
 694                         M_COPY(src,       iptr->dst);
 695                         M_COPY(src->prev, iptr->dst->prev);
 696                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 697                         break;
 698
 699                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 700
 701                         M_COPY(src,             iptr->dst);
 702                         M_COPY(src->prev,       iptr->dst->prev);
 703                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 704                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 705                         break;
 706
 707                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 708
 709                         M_COPY(src,       iptr->dst);
 710                         M_COPY(src->prev, iptr->dst->prev);
 711                         break;
 712
 713                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 714
 715                         M_COPY(src,             iptr->dst);
 716                         M_COPY(src->prev,       iptr->dst->prev);
 717                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 718                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 719                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 720                         break;
 721
 722                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 723
 724                         M_COPY(src,                   iptr->dst);
 725                         M_COPY(src->prev,             iptr->dst->prev);
 726                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 727                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 728                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 729                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 730                         break;
 731
 732                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 733
 734                         M_COPY(src,       iptr->dst->prev);
 735                         M_COPY(src->prev, iptr->dst);
 736                         break;
 737
 738
 739                 /* integer operations *************************************************/
 740
 741                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 742
 743                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 744                         if (iptr->dst->flags & INMEMORY) {
 745                                 if (src->flags & INMEMORY) {
 746                                         if (src->regoff == iptr->dst->regoff) {
 747                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 748
 749                                         } else {
 750                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 751                                                 x86_64_negl_reg(cd, REG_ITMP1);
 752                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 753                                         }
 754
 755                                 } else {
 756                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 757                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 758                                 }
 759
 760                         } else {
 761                                 if (src->flags & INMEMORY) {
 762                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 763                                         x86_64_negl_reg(cd, d);
 764
 765                                 } else {
 766                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 767                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 768                                 }
 769                         }
 770                         break;
 771
 772                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 773
 774                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 775                         if (iptr->dst->flags & INMEMORY) {
 776                                 if (src->flags & INMEMORY) {
 777                                         if (src->regoff == iptr->dst->regoff) {
 778                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 779
 780                                         } else {
 781                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 782                                                 x86_64_neg_reg(cd, REG_ITMP1);
 783                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 784                                         }
 785
 786                                 } else {
 787                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 788                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 789                                 }
 790
 791                         } else {
 792                                 if (src->flags & INMEMORY) {
 793                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 794                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 795
 796                                 } else {
 797                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 798                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 799                                 }
 800                         }
 801                         break;
 802
 803                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 804
 805                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 806                         if (src->flags & INMEMORY) {
 807                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 808
 809                         } else {
 810                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 811                         }
 812                         store_reg_to_var_int(iptr->dst, d);
 813                         break;
 814
 815                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 816
 817                         var_to_reg_int(s1, src, REG_ITMP1);
 818                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 819                         M_INTMOVE(s1, d);
 820                         store_reg_to_var_int(iptr->dst, d);
 821                         break;
 822
 823                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 824
 825                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 826                         if (src->flags & INMEMORY) {
 827                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 828
 829                         } else {
 830                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 831                         }
 832                         store_reg_to_var_int(iptr->dst, d);
 833                         break;
 834
 835                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 836
 837                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 838                         if (src->flags & INMEMORY) {
 839                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 840
 841                         } else {
 842                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 843                         }
 844                         store_reg_to_var_int(iptr->dst, d);
 845                         break;
 846
 847                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 848
 849                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 850                         if (src->flags & INMEMORY) {
 851                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 852
 853                         } else {
 854                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 855                         }
 856                         store_reg_to_var_int(iptr->dst, d);
 857                         break;
 858
 859
 860                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 861
 862                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 863                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 864                         break;
 865
 866                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 867                                       /* val.i = constant                             */
 868
 869                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 870                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 871                         break;
 872
 873                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 874
 875                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 876                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 877                         break;
 878
 879                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 880                                       /* val.l = constant                             */
 881
 882                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 883                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 884                         break;
 885
 886                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 887
 888                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 889                         if (iptr->dst->flags & INMEMORY) {
 890                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 891                                         if (src->prev->regoff == iptr->dst->regoff) {
 892                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 893                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 894
 895                                         } else {
 896                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 897                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 898                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 899                                         }
 900
 901                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 902                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 903                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 904                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 905
 906                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 907                                         if (src->prev->regoff == iptr->dst->regoff) {
 908                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 909
 910                                         } else {
 911                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 912                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 913                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 914                                         }
 915
 916                                 } else {
 917                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 918                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 919                                 }
 920
 921                         } else {
 922                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 923                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 924                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 925
 926                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 927                                         M_INTMOVE(src->prev->regoff, d);
 928                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 929
 930                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 931                                         /* workaround for reg alloc */
 932                                         if (src->regoff == iptr->dst->regoff) {
 933                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 934                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 935                                                 M_INTMOVE(REG_ITMP1, d);
 936
 937                                         } else {
 938                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 939                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 940                                         }
 941
 942                                 } else {
 943                                         /* workaround for reg alloc */
 944                                         if (src->regoff == iptr->dst->regoff) {
 945                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 946                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 947                                                 M_INTMOVE(REG_ITMP1, d);
 948
 949                                         } else {
 950                                                 M_INTMOVE(src->prev->regoff, d);
 951                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 952                                         }
 953                                 }
 954                         }
 955                         break;
 956
 957                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 958                                       /* val.i = constant                             */
 959
 960                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 961                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 962                         break;
 963
 964                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 965
 966                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 967                         if (iptr->dst->flags & INMEMORY) {
 968                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 969                                         if (src->prev->regoff == iptr->dst->regoff) {
 970                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 971                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 972
 973                                         } else {
 974                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 975                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 976                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 977                                         }
 978
 979                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 980                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 981                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 982                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 983
 984                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 985                                         if (src->prev->regoff == iptr->dst->regoff) {
 986                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 987
 988                                         } else {
 989                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 990                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 991                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 992                                         }
 993
 994                                 } else {
 995                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 996                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 997                                 }
 998
 999                         } else {
1000                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1001                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1002                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1003
1004                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1005                                         M_INTMOVE(src->prev->regoff, d);
1006                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1007
1008                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1009                                         /* workaround for reg alloc */
1010                                         if (src->regoff == iptr->dst->regoff) {
1011                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1012                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1013                                                 M_INTMOVE(REG_ITMP1, d);
1014
1015                                         } else {
1016                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1017                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1018                                         }
1019
1020                                 } else {
1021                                         /* workaround for reg alloc */
1022                                         if (src->regoff == iptr->dst->regoff) {
1023                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1024                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1025                                                 M_INTMOVE(REG_ITMP1, d);
1026
1027                                         } else {
1028                                                 M_INTMOVE(src->prev->regoff, d);
1029                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1030                                         }
1031                                 }
1032                         }
1033                         break;
1034
1035                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1036                                       /* val.l = constant                             */
1037
1038                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1039                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1040                         break;
1041
1042                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1043
1044                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1045                         if (iptr->dst->flags & INMEMORY) {
1046                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1047                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1048                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1049                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1050
1051                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1052                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1053                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1054                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1055
1056                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1057                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1058                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1059                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1060
1061                                 } else {
1062                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1063                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1064                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1065                                 }
1066
1067                         } else {
1068                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1069                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1070                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1071
1072                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1073                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1074                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1075
1076                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1077                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1078                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1079
1080                                 } else {
1081                                         if (src->regoff == iptr->dst->regoff) {
1082                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1083
1084                                         } else {
1085                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1086                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1087                                         }
1088                                 }
1089                         }
1090                         break;
1091
1092                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1093                                       /* val.i = constant                             */
1094
1095                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1096                         if (iptr->dst->flags & INMEMORY) {
1097                                 if (src->flags & INMEMORY) {
1098                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1099                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1100
1101                                 } else {
1102                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1103                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1104                                 }
1105
1106                         } else {
1107                                 if (src->flags & INMEMORY) {
1108                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1109
1110                                 } else {
1111                                         if (iptr->val.i == 2) {
1112                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1113                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1114
1115                                         } else {
1116                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1117                                         }
1118                                 }
1119                         }
1120                         break;
1121
1122                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1123
1124                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1125                         if (iptr->dst->flags & INMEMORY) {
1126                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1127                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1128                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1129                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1130
1131                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1132                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1133                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1134                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1135
1136                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1137                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1138                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1139                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1140
1141                                 } else {
1142                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1143                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1144                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1145                                 }
1146
1147                         } else {
1148                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1149                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1150                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1151
1152                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1153                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1154                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1155
1156                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1157                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1158                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1159
1160                                 } else {
1161                                         if (src->regoff == iptr->dst->regoff) {
1162                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1163
1164                                         } else {
1165                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1166                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1167                                         }
1168                                 }
1169                         }
1170                         break;
1171
1172                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1173                                       /* val.l = constant                             */
1174
1175                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1176                         if (iptr->dst->flags & INMEMORY) {
1177                                 if (src->flags & INMEMORY) {
1178                                         if (IS_IMM32(iptr->val.l)) {
1179                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1180
1181                                         } else {
1182                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1183                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1184                                         }
1185                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1186
1187                                 } else {
1188                                         if (IS_IMM32(iptr->val.l)) {
1189                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1190
1191                                         } else {
1192                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1193                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1194                                         }
1195                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1196                                 }
1197
1198                         } else {
1199                                 if (src->flags & INMEMORY) {
1200                                         if (IS_IMM32(iptr->val.l)) {
1201                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1202
1203                                         } else {
1204                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1205                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1206                                         }
1207
1208                                 } else {
1209                                         /* should match in many cases */
1210                                         if (iptr->val.l == 2) {
1211                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1212                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1213
1214                                         } else {
1215                                                 if (IS_IMM32(iptr->val.l)) {
1216                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1217
1218                                                 } else {
1219                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1220                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1221                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1222                                                 }
1223                                         }
1224                                 }
1225                         }
1226                         break;
1227
1228                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1229
1230                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1231                 if (src->prev->flags & INMEMORY) {
1232                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1233
1234                         } else {
1235                                 M_INTMOVE(src->prev->regoff, RAX);
1236                         }
1237
1238                         if (src->flags & INMEMORY) {
1239                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1240
1241                         } else {
1242                                 M_INTMOVE(src->regoff, REG_ITMP3);
1243                         }
1244                         gen_div_check(src);
1245
1246                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1247                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1248                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1249                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1250
1251                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1252                         x86_64_cltd(cd);
1253                         x86_64_idivl_reg(cd, REG_ITMP3);
1254
1255                         if (iptr->dst->flags & INMEMORY) {
1256                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1257                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1258
1259                         } else {
1260                                 M_INTMOVE(RAX, iptr->dst->regoff);
1261
1262                                 if (iptr->dst->regoff != RDX) {
1263                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1264                                 }
1265                         }
1266                         break;
1267
1268                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1269                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1270                         if (src->prev->flags & INMEMORY) {
1271                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1272
1273                         } else {
1274                                 M_INTMOVE(src->prev->regoff, RAX);
1275                         }
1276
1277                         if (src->flags & INMEMORY) {
1278                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1279
1280                         } else {
1281                                 M_INTMOVE(src->regoff, REG_ITMP3);
1282                         }
1283                         gen_div_check(src);
1284
1285                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1286
1287                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1288                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1289
1290
1291                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1292                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1293                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1294
1295                         x86_64_cltd(cd);
1296                         x86_64_idivl_reg(cd, REG_ITMP3);
1297
1298                         if (iptr->dst->flags & INMEMORY) {
1299                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1300                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1301
1302                         } else {
1303                                 M_INTMOVE(RDX, iptr->dst->regoff);
1304
1305                                 if (iptr->dst->regoff != RDX) {
1306                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1307                                 }
1308                         }
1309                         break;
1310
1311                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1312                                       /* val.i = constant                             */
1313
1314                         var_to_reg_int(s1, src, REG_ITMP1);
1315                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1316                         M_INTMOVE(s1, REG_ITMP1);
1317                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1318                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1319                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1320                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1321                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1322                         store_reg_to_var_int(iptr->dst, d);
1323                         break;
1324
1325                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1326                                       /* val.i = constant                             */
1327
1328                         var_to_reg_int(s1, src, REG_ITMP1);
1329                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1330                         M_INTMOVE(s1, REG_ITMP1);
1331                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1332                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1333                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1334                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1335                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1336                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1337                         store_reg_to_var_int(iptr->dst, d);
1338                         break;
1339
1340
1341                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1342
1343                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1344
1345                 if (src->prev->flags & INMEMORY) {
1346                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1347
1348                         } else {
1349                                 M_INTMOVE(src->prev->regoff, RAX);
1350                         }
1351
1352                         if (src->flags & INMEMORY) {
1353                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1354
1355                         } else {
1356                                 M_INTMOVE(src->regoff, REG_ITMP3);
1357                         }
1358                         gen_div_check(src);
1359
1360                         /* check as described in jvm spec */
1361                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1362                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1363                         M_BNE(4 + 6);
1364                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1365                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1366
1367                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1368                         x86_64_cqto(cd);
1369                         x86_64_idiv_reg(cd, REG_ITMP3);
1370
1371                         if (iptr->dst->flags & INMEMORY) {
1372                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1373                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1374
1375                         } else {
1376                                 M_INTMOVE(RAX, iptr->dst->regoff);
1377
1378                                 if (iptr->dst->regoff != RDX) {
1379                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1380                                 }
1381                         }
1382                         break;
1383
1384                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1385
1386                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1387                         if (src->prev->flags & INMEMORY) {
1388                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1389
1390                         } else {
1391                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1392                         }
1393
1394                         if (src->flags & INMEMORY) {
1395                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1396
1397                         } else {
1398                                 M_INTMOVE(src->regoff, REG_ITMP3);
1399                         }
1400                         gen_div_check(src);
1401
1402                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1403
1404                         /* check as described in jvm spec */
1405                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1406                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1407                         M_BNE(3 + 4 + 6);
1408
1409 #if 0
1410                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1411 #endif
1412                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1413                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1414                         M_BEQ(2 + 3);                                          /* 6 bytes */
1415
1416                         x86_64_cqto(cd);
1417                         x86_64_idiv_reg(cd, REG_ITMP3);
1418
1419                         if (iptr->dst->flags & INMEMORY) {
1420                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1421                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1422
1423                         } else {
1424                                 M_INTMOVE(RDX, iptr->dst->regoff);
1425
1426                                 if (iptr->dst->regoff != RDX) {
1427                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1428                                 }
1429                         }
1430                         break;
1431
1432                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1433                                       /* val.i = constant                             */
1434
1435                         var_to_reg_int(s1, src, REG_ITMP1);
1436                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1437                         M_INTMOVE(s1, REG_ITMP1);
1438                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1439                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1440                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1441                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1442                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1443                         store_reg_to_var_int(iptr->dst, d);
1444                         break;
1445
1446                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1447                                       /* val.l = constant                             */
1448
1449                         var_to_reg_int(s1, src, REG_ITMP1);
1450                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1451                         M_INTMOVE(s1, REG_ITMP1);
1452                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1453                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1454                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1455                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1456                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1457                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1458                         store_reg_to_var_int(iptr->dst, d);
1459                         break;
1460
1461                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1462
1463                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1464                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1465                         break;
1466
1467                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1468                                       /* val.i = constant                             */
1469
1470                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1471                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1472                         break;
1473
1474                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1475
1476                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1477                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1478                         break;
1479
1480                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1481                                       /* val.i = constant                             */
1482
1483                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1484                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1485                         break;
1486
1487                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1488
1489                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1490                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1491                         break;
1492
1493                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1494                                       /* val.i = constant                             */
1495
1496                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1497                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1498                         break;
1499
1500                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1501
1502                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1503                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1504                         break;
1505
1506         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1507                                           /* val.i = constant                             */
1508
1509                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1510                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1511                         break;
1512
1513                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1514
1515                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1516                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1517                         break;
1518
1519                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1520                                       /* val.i = constant                             */
1521
1522                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1523                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1524                         break;
1525
1526                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1527
1528                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1529                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1530                         break;
1531
1532                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1533                                       /* val.l = constant                             */
1534
1535                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1536                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1537                         break;
1538
1539                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1540
1541                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1542                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1543                         break;
1544
1545                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1546                                       /* val.i = constant                             */
1547
1548                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1549                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1550                         break;
1551
1552                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1553
1554                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1555                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1556                         break;
1557
1558                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1559                                       /* val.l = constant                             */
1560
1561                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1562                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1563                         break;
1564
1565                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1566
1567                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1568                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1569                         break;
1570
1571                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1572                                       /* val.i = constant                             */
1573
1574                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1575                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1576                         break;
1577
1578                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1579
1580                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1581                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1582                         break;
1583
1584                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1585                                       /* val.l = constant                             */
1586
1587                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1588                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1589                         break;
1590
1591                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1592
1593                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1594                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1595                         break;
1596
1597                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1598                                       /* val.i = constant                             */
1599
1600                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1601                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1602                         break;
1603
1604                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1605
1606                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1607                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1608                         break;
1609
1610                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1611                                       /* val.l = constant                             */
1612
1613                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1614                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1615                         break;
1616
1617
1618                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1619                                       /* op1 = variable, val.i = constant             */
1620
1621                         /* using inc and dec is definitely faster than add -- tested      */
1622                         /* with sieve                                                     */
1623
1624                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1625                         d = var->regoff;
1626                         if (var->flags & INMEMORY) {
1627                                 if (iptr->val.i == 1) {
1628                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1629
1630                                 } else if (iptr->val.i == -1) {
1631                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1632
1633                                 } else {
1634                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1635                                 }
1636
1637                         } else {
1638                                 if (iptr->val.i == 1) {
1639                                         x86_64_incl_reg(cd, d);
1640
1641                                 } else if (iptr->val.i == -1) {
1642                                         x86_64_decl_reg(cd, d);
1643
1644                                 } else {
1645                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1646                                 }
1647                         }
1648                         break;
1649
1650
1651                 /* floating operations ************************************************/
1652
1653                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1654
1655                         var_to_reg_flt(s1, src, REG_FTMP1);
1656                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1657                         disp = dseg_adds4(cd, 0x80000000);
1658                         M_FLTMOVE(s1, d);
1659                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1660                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1661                         store_reg_to_var_flt(iptr->dst, d);
1662                         break;
1663
1664                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1665
1666                         var_to_reg_flt(s1, src, REG_FTMP1);
1667                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1668                         disp = dseg_adds8(cd, 0x8000000000000000);
1669                         M_FLTMOVE(s1, d);
1670                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1671                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1672                         store_reg_to_var_flt(iptr->dst, d);
1673                         break;
1674
1675                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1676
1677                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1678                         var_to_reg_flt(s2, src, REG_FTMP2);
1679                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1680                         if (s1 == d) {
1681                                 x86_64_addss_reg_reg(cd, s2, d);
1682                         } else if (s2 == d) {
1683                                 x86_64_addss_reg_reg(cd, s1, d);
1684                         } else {
1685                                 M_FLTMOVE(s1, d);
1686                                 x86_64_addss_reg_reg(cd, s2, d);
1687                         }
1688                         store_reg_to_var_flt(iptr->dst, d);
1689                         break;
1690
1691                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1692
1693                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1694                         var_to_reg_flt(s2, src, REG_FTMP2);
1695                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1696                         if (s1 == d) {
1697                                 x86_64_addsd_reg_reg(cd, s2, d);
1698                         } else if (s2 == d) {
1699                                 x86_64_addsd_reg_reg(cd, s1, d);
1700                         } else {
1701                                 M_FLTMOVE(s1, d);
1702                                 x86_64_addsd_reg_reg(cd, s2, d);
1703                         }
1704                         store_reg_to_var_flt(iptr->dst, d);
1705                         break;
1706
1707                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1708
1709                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1710                         var_to_reg_flt(s2, src, REG_FTMP2);
1711                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1712                         if (s2 == d) {
1713                                 M_FLTMOVE(s2, REG_FTMP2);
1714                                 s2 = REG_FTMP2;
1715                         }
1716                         M_FLTMOVE(s1, d);
1717                         x86_64_subss_reg_reg(cd, s2, d);
1718                         store_reg_to_var_flt(iptr->dst, d);
1719                         break;
1720
1721                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1722
1723                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1724                         var_to_reg_flt(s2, src, REG_FTMP2);
1725                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1726                         if (s2 == d) {
1727                                 M_FLTMOVE(s2, REG_FTMP2);
1728                                 s2 = REG_FTMP2;
1729                         }
1730                         M_FLTMOVE(s1, d);
1731                         x86_64_subsd_reg_reg(cd, s2, d);
1732                         store_reg_to_var_flt(iptr->dst, d);
1733                         break;
1734
1735                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1736
1737                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1738                         var_to_reg_flt(s2, src, REG_FTMP2);
1739                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1740                         if (s1 == d) {
1741                                 x86_64_mulss_reg_reg(cd, s2, d);
1742                         } else if (s2 == d) {
1743                                 x86_64_mulss_reg_reg(cd, s1, d);
1744                         } else {
1745                                 M_FLTMOVE(s1, d);
1746                                 x86_64_mulss_reg_reg(cd, s2, d);
1747                         }
1748                         store_reg_to_var_flt(iptr->dst, d);
1749                         break;
1750
1751                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1752
1753                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1754                         var_to_reg_flt(s2, src, REG_FTMP2);
1755                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1756                         if (s1 == d) {
1757                                 x86_64_mulsd_reg_reg(cd, s2, d);
1758                         } else if (s2 == d) {
1759                                 x86_64_mulsd_reg_reg(cd, s1, d);
1760                         } else {
1761                                 M_FLTMOVE(s1, d);
1762                                 x86_64_mulsd_reg_reg(cd, s2, d);
1763                         }
1764                         store_reg_to_var_flt(iptr->dst, d);
1765                         break;
1766
1767                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1768
1769                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1770                         var_to_reg_flt(s2, src, REG_FTMP2);
1771                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1772                         if (s2 == d) {
1773                                 M_FLTMOVE(s2, REG_FTMP2);
1774                                 s2 = REG_FTMP2;
1775                         }
1776                         M_FLTMOVE(s1, d);
1777                         x86_64_divss_reg_reg(cd, s2, d);
1778                         store_reg_to_var_flt(iptr->dst, d);
1779                         break;
1780
1781                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1782
1783                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1784                         var_to_reg_flt(s2, src, REG_FTMP2);
1785                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1786                         if (s2 == d) {
1787                                 M_FLTMOVE(s2, REG_FTMP2);
1788                                 s2 = REG_FTMP2;
1789                         }
1790                         M_FLTMOVE(s1, d);
1791                         x86_64_divsd_reg_reg(cd, s2, d);
1792                         store_reg_to_var_flt(iptr->dst, d);
1793                         break;
1794
1795                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1796
1797                         var_to_reg_int(s1, src, REG_ITMP1);
1798                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1799                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1800                         store_reg_to_var_flt(iptr->dst, d);
1801                         break;
1802
1803                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1804
1805                         var_to_reg_int(s1, src, REG_ITMP1);
1806                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1807                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1808                         store_reg_to_var_flt(iptr->dst, d);
1809                         break;
1810
1811                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1812
1813                         var_to_reg_int(s1, src, REG_ITMP1);
1814                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1815                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1816                         store_reg_to_var_flt(iptr->dst, d);
1817                         break;
1818
1819                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1820
1821                         var_to_reg_int(s1, src, REG_ITMP1);
1822                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1823                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1824                         store_reg_to_var_flt(iptr->dst, d);
1825                         break;
1826
1827                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1828
1829                         var_to_reg_flt(s1, src, REG_FTMP1);
1830                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1831                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1832                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1833                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1834                         x86_64_jcc(cd, X86_64_CC_NE, a);
1835                         M_FLTMOVE(s1, REG_FTMP1);
1836                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1837                         x86_64_call_reg(cd, REG_ITMP2);
1838                         M_INTMOVE(REG_RESULT, d);
1839                         store_reg_to_var_int(iptr->dst, d);
1840                         break;
1841
1842                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1843
1844                         var_to_reg_flt(s1, src, REG_FTMP1);
1845                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1846                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1847                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1848                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1849                         x86_64_jcc(cd, X86_64_CC_NE, a);
1850                         M_FLTMOVE(s1, REG_FTMP1);
1851                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1852                         x86_64_call_reg(cd, REG_ITMP2);
1853                         M_INTMOVE(REG_RESULT, d);
1854                         store_reg_to_var_int(iptr->dst, d);
1855                         break;
1856
1857                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1858
1859                         var_to_reg_flt(s1, src, REG_FTMP1);
1860                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1861                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1862                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1863                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1864                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1865                         x86_64_jcc(cd, X86_64_CC_NE, a);
1866                         M_FLTMOVE(s1, REG_FTMP1);
1867                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1868                         x86_64_call_reg(cd, REG_ITMP2);
1869                         M_INTMOVE(REG_RESULT, d);
1870                         store_reg_to_var_int(iptr->dst, d);
1871                         break;
1872
1873                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1874
1875                         var_to_reg_flt(s1, src, REG_FTMP1);
1876                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1877                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1878                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1879                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1880                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1881                         x86_64_jcc(cd, X86_64_CC_NE, a);
1882                         M_FLTMOVE(s1, REG_FTMP1);
1883                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1884                         x86_64_call_reg(cd, REG_ITMP2);
1885                         M_INTMOVE(REG_RESULT, d);
1886                         store_reg_to_var_int(iptr->dst, d);
1887                         break;
1888
1889                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1890
1891                         var_to_reg_flt(s1, src, REG_FTMP1);
1892                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1893                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1894                         store_reg_to_var_flt(iptr->dst, d);
1895                         break;
1896
1897                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1898
1899                         var_to_reg_flt(s1, src, REG_FTMP1);
1900                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1901                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1902                         store_reg_to_var_flt(iptr->dst, d);
1903                         break;
1904
1905                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1906                                           /* == => 0, < => 1, > => -1 */
1907
1908                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1909                         var_to_reg_flt(s2, src, REG_FTMP2);
1910                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1911                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1912                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1913                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1914                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1915                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1916                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1917                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1918                         store_reg_to_var_int(iptr->dst, d);
1919                         break;
1920
1921                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1922                                           /* == => 0, < => 1, > => -1 */
1923
1924                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1925                         var_to_reg_flt(s2, src, REG_FTMP2);
1926                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1927                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1928                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1929                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1930                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1931                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1932                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1933                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1934                         store_reg_to_var_int(iptr->dst, d);
1935                         break;
1936
1937                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1938                                           /* == => 0, < => 1, > => -1 */
1939
1940                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1941                         var_to_reg_flt(s2, src, REG_FTMP2);
1942                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1943                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1944                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1945                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1946                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1947                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1948                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1949                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1950                         store_reg_to_var_int(iptr->dst, d);
1951                         break;
1952
1953                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1954                                           /* == => 0, < => 1, > => -1 */
1955
1956                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1957                         var_to_reg_flt(s2, src, REG_FTMP2);
1958                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1959                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1960                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1961                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1962                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1963                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1964                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1965                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1966                         store_reg_to_var_int(iptr->dst, d);
1967                         break;
1968
1969
1970                 /* memory operations **************************************************/
1971
1972                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1973
1974                         var_to_reg_int(s1, src, REG_ITMP1);
1975                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1976                         gen_nullptr_check(s1);
1977                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1978                         store_reg_to_var_int(iptr->dst, d);
1979                         break;
1980
1981                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1982
1983                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1984                         var_to_reg_int(s2, src, REG_ITMP2);
1985                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1986                         if (iptr->op1 == 0) {
1987                                 gen_nullptr_check(s1);
1988                                 gen_bound_check;
1989                         }
1990                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1991                         store_reg_to_var_int(iptr->dst, d);
1992                         break;
1993
1994                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1995
1996                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1997                         var_to_reg_int(s2, src, REG_ITMP2);
1998                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1999                         if (iptr->op1 == 0) {
2000                                 gen_nullptr_check(s1);
2001                                 gen_bound_check;
2002                         }
2003                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2004                         store_reg_to_var_int(iptr->dst, d);
2005                         break;
2006
2007                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2008
2009                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2010                         var_to_reg_int(s2, src, REG_ITMP2);
2011                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2012                         if (iptr->op1 == 0) {
2013                                 gen_nullptr_check(s1);
2014                                 gen_bound_check;
2015                         }
2016                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2017                         store_reg_to_var_int(iptr->dst, d);
2018                         break;
2019
2020                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2021
2022                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2023                         var_to_reg_int(s2, src, REG_ITMP2);
2024                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2025                         if (iptr->op1 == 0) {
2026                                 gen_nullptr_check(s1);
2027                                 gen_bound_check;
2028                         }
2029                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2030                         store_reg_to_var_flt(iptr->dst, d);
2031                         break;
2032
2033                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2034
2035                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2036                         var_to_reg_int(s2, src, REG_ITMP2);
2037                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2038                         if (iptr->op1 == 0) {
2039                                 gen_nullptr_check(s1);
2040                                 gen_bound_check;
2041                         }
2042                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2043                         store_reg_to_var_flt(iptr->dst, d);
2044                         break;
2045
2046                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2047
2048                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2049                         var_to_reg_int(s2, src, REG_ITMP2);
2050                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2051                         if (iptr->op1 == 0) {
2052                                 gen_nullptr_check(s1);
2053                                 gen_bound_check;
2054                         }
2055                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2056                         store_reg_to_var_int(iptr->dst, d);
2057                         break;
2058
2059                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2060
2061                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2062                         var_to_reg_int(s2, src, REG_ITMP2);
2063                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2064                         if (iptr->op1 == 0) {
2065                                 gen_nullptr_check(s1);
2066                                 gen_bound_check;
2067                         }
2068                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2069                         store_reg_to_var_int(iptr->dst, d);
2070                         break;
2071
2072                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2073
2074                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2075                         var_to_reg_int(s2, src, REG_ITMP2);
2076                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2077                         if (iptr->op1 == 0) {
2078                                 gen_nullptr_check(s1);
2079                                 gen_bound_check;
2080                         }
2081                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2082                         store_reg_to_var_int(iptr->dst, d);
2083                         break;
2084
2085
2086                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2087
2088                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2089                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2090                         if (iptr->op1 == 0) {
2091                                 gen_nullptr_check(s1);
2092                                 gen_bound_check;
2093                         }
2094                         var_to_reg_int(s3, src, REG_ITMP3);
2095                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2096                         break;
2097
2098                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2099
2100                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2101                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2102                         if (iptr->op1 == 0) {
2103                                 gen_nullptr_check(s1);
2104                                 gen_bound_check;
2105                         }
2106                         var_to_reg_int(s3, src, REG_ITMP3);
2107                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2108                         break;
2109
2110                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2111
2112                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2113                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2114                         if (iptr->op1 == 0) {
2115                                 gen_nullptr_check(s1);
2116                                 gen_bound_check;
2117                         }
2118                         var_to_reg_flt(s3, src, REG_FTMP3);
2119                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2120                         break;
2121
2122                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2123
2124                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2125                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2126                         if (iptr->op1 == 0) {
2127                                 gen_nullptr_check(s1);
2128                                 gen_bound_check;
2129                         }
2130                         var_to_reg_flt(s3, src, REG_FTMP3);
2131                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2132                         break;
2133
2134                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2135
2136                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2137                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2138                         if (iptr->op1 == 0) {
2139                                 gen_nullptr_check(s1);
2140                                 gen_bound_check;
2141                         }
2142                         var_to_reg_int(s3, src, REG_ITMP3);
2143                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2144                         break;
2145
2146                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2147
2148                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2149                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2150                         if (iptr->op1 == 0) {
2151                                 gen_nullptr_check(s1);
2152                                 gen_bound_check;
2153                         }
2154                         var_to_reg_int(s3, src, REG_ITMP3);
2155                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2156                         break;
2157
2158                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2159
2160                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2161                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2162                         if (iptr->op1 == 0) {
2163                                 gen_nullptr_check(s1);
2164                                 gen_bound_check;
2165                         }
2166                         var_to_reg_int(s3, src, REG_ITMP3);
2167                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2168                         break;
2169
2170                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2171
2172                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2173                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2174                         if (iptr->op1 == 0) {
2175                                 gen_nullptr_check(s1);
2176                                 gen_bound_check;
2177                         }
2178                         var_to_reg_int(s3, src, REG_ITMP3);
2179
2180                         M_MOV(s1, rd->argintregs[0]);
2181                         M_MOV(s3, rd->argintregs[1]);
2182                         M_MOV_IMM((ptrint) BUILTIN_canstore, REG_ITMP1);
2183                         M_CALL(REG_ITMP1);
2184                         M_TEST(REG_RESULT);
2185                         M_BEQ(0);
2186                         codegen_addxstorerefs(cd, cd->mcodeptr);
2187
2188                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2189                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2190                         var_to_reg_int(s3, src, REG_ITMP3);
2191                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2192                         break;
2193
2194
2195                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2196
2197                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2198                         var_to_reg_int(s2, src, REG_ITMP2);
2199                         if (iptr->op1 == 0) {
2200                                 gen_nullptr_check(s1);
2201                                 gen_bound_check;
2202                         }
2203                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2204                         break;
2205
2206                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2207
2208                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2209                         var_to_reg_int(s2, src, REG_ITMP2);
2210                         if (iptr->op1 == 0) {
2211                                 gen_nullptr_check(s1);
2212                                 gen_bound_check;
2213                         }
2214
2215                         if (IS_IMM32(iptr->val.l)) {
2216                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2217
2218                         } else {
2219                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2220                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2221                         }
2222                         break;
2223
2224                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2225
2226                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2227                         var_to_reg_int(s2, src, REG_ITMP2);
2228                         if (iptr->op1 == 0) {
2229                                 gen_nullptr_check(s1);
2230                                 gen_bound_check;
2231                         }
2232                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2233                         break;
2234
2235                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2236
2237                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2238                         var_to_reg_int(s2, src, REG_ITMP2);
2239                         if (iptr->op1 == 0) {
2240                                 gen_nullptr_check(s1);
2241                                 gen_bound_check;
2242                         }
2243                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2244                         break;
2245
2246                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2247
2248                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2249                         var_to_reg_int(s2, src, REG_ITMP2);
2250                         if (iptr->op1 == 0) {
2251                                 gen_nullptr_check(s1);
2252                                 gen_bound_check;
2253                         }
2254                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2255                         break;
2256
2257                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2258
2259                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2260                         var_to_reg_int(s2, src, REG_ITMP2);
2261                         if (iptr->op1 == 0) {
2262                                 gen_nullptr_check(s1);
2263                                 gen_bound_check;
2264                         }
2265                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2266                         break;
2267
2268
2269                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2270                                       /* op1 = type, val.a = field address            */
2271
2272                         if (!iptr->val.a) {
2273                                 disp = dseg_addaddress(cd, NULL);
2274
2275                                 codegen_addpatchref(cd, cd->mcodeptr,
2276                                                                         PATCHER_get_putstatic,
2277                                                                         (unresolved_field *) iptr->target, disp);
2278
2279                                 if (opt_showdisassemble) {
2280                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2281                                 }
2282
2283                         } else {
2284                                 fieldinfo *fi = iptr->val.a;
2285
2286                                 disp = dseg_addaddress(cd, &(fi->value));
2287
2288                                 if (!fi->class->initialized) {
2289                                         codegen_addpatchref(cd, cd->mcodeptr,
2290                                                                                 PATCHER_clinit, fi->class, 0);
2291
2292                                         if (opt_showdisassemble) {
2293                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2294                                         }
2295                                 }
2296                         }
2297
2298                         /* This approach is much faster than moving the field address     */
2299                         /* inline into a register. */
2300                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2301                         switch (iptr->op1) {
2302                         case TYPE_INT:
2303                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2304                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2305                                 store_reg_to_var_int(iptr->dst, d);
2306                                 break;
2307                         case TYPE_LNG:
2308                         case TYPE_ADR:
2309                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2310                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2311                                 store_reg_to_var_int(iptr->dst, d);
2312                                 break;
2313                         case TYPE_FLT:
2314                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2315                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2316                                 store_reg_to_var_flt(iptr->dst, d);
2317                                 break;
2318                         case TYPE_DBL:
2319                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2320                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2321                                 store_reg_to_var_flt(iptr->dst, d);
2322                                 break;
2323                         }
2324                         break;
2325
2326                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2327                                       /* op1 = type, val.a = field address            */
2328
2329                         if (!iptr->val.a) {
2330                                 disp = dseg_addaddress(cd, NULL);
2331
2332                                 codegen_addpatchref(cd, cd->mcodeptr,
2333                                                                         PATCHER_get_putstatic,
2334                                                                         (unresolved_field *) iptr->target, disp);
2335
2336                                 if (opt_showdisassemble) {
2337                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2338                                 }
2339
2340                         } else {
2341                                 fieldinfo *fi = iptr->val.a;
2342
2343                                 disp = dseg_addaddress(cd, &(fi->value));
2344
2345                                 if (!fi->class->initialized) {
2346                                         codegen_addpatchref(cd, cd->mcodeptr,
2347                                                                                 PATCHER_clinit, fi->class, 0);
2348
2349                                         if (opt_showdisassemble) {
2350                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2351                                         }
2352                                 }
2353                         }
2354
2355                         /* This approach is much faster than moving the field address     */
2356                         /* inline into a register. */
2357                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2358                         switch (iptr->op1) {
2359                         case TYPE_INT:
2360                                 var_to_reg_int(s2, src, REG_ITMP1);
2361                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2362                                 break;
2363                         case TYPE_LNG:
2364                         case TYPE_ADR:
2365                                 var_to_reg_int(s2, src, REG_ITMP1);
2366                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2367                                 break;
2368                         case TYPE_FLT:
2369                                 var_to_reg_flt(s2, src, REG_FTMP1);
2370                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2371                                 break;
2372                         case TYPE_DBL:
2373                                 var_to_reg_flt(s2, src, REG_FTMP1);
2374                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2375                                 break;
2376                         }
2377                         break;
2378
2379                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2380                                           /* val = value (in current instruction)     */
2381                                           /* op1 = type, val.a = field address (in    */
2382                                           /* following NOP)                           */
2383
2384                         if (!iptr[1].val.a) {
2385                                 disp = dseg_addaddress(cd, NULL);
2386
2387                                 codegen_addpatchref(cd, cd->mcodeptr,
2388                                                                         PATCHER_get_putstatic,
2389                                                                         (unresolved_field *) iptr[1].target, disp);
2390
2391                                 if (opt_showdisassemble) {
2392                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2393                                 }
2394
2395                         } else {
2396                                 fieldinfo *fi = iptr[1].val.a;
2397
2398                                 disp = dseg_addaddress(cd, &(fi->value));
2399
2400                                 if (!fi->class->initialized) {
2401                                         codegen_addpatchref(cd, cd->mcodeptr,
2402                                                                                 PATCHER_clinit, fi->class, 0);
2403
2404                                         if (opt_showdisassemble) {
2405                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2406                                         }
2407                                 }
2408                         }
2409
2410                         /* This approach is much faster than moving the field address     */
2411                         /* inline into a register. */
2412                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
2413                         switch (iptr->op1) {
2414                         case TYPE_INT:
2415                         case TYPE_FLT:
2416                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2417                                 break;
2418                         case TYPE_LNG:
2419                         case TYPE_ADR:
2420                         case TYPE_DBL:
2421                                 if (IS_IMM32(iptr->val.l)) {
2422                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2423                                 } else {
2424                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2425                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2426                                 }
2427                                 break;
2428                         }
2429                         break;
2430
2431                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2432                                       /* op1 = type, val.i = field offset             */
2433
2434                         var_to_reg_int(s1, src, REG_ITMP1);
2435                         gen_nullptr_check(s1);
2436
2437                         if (!iptr->val.a) {
2438                                 codegen_addpatchref(cd, cd->mcodeptr,
2439                                                                         PATCHER_get_putfield,
2440                                                                         (unresolved_field *) iptr->target, 0);
2441
2442                                 if (opt_showdisassemble) {
2443                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2444                                 }
2445
2446                                 a = 0;
2447
2448                         } else {
2449                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2450                         }
2451
2452                         switch (iptr->op1) {
2453                         case TYPE_INT:
2454                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2455                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2456                                 store_reg_to_var_int(iptr->dst, d);
2457                                 break;
2458                         case TYPE_LNG:
2459                         case TYPE_ADR:
2460                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2461                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2462                                 store_reg_to_var_int(iptr->dst, d);
2463                                 break;
2464                         case TYPE_FLT:
2465                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2466                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2467                                 store_reg_to_var_flt(iptr->dst, d);
2468                                 break;
2469                         case TYPE_DBL:
2470                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2471                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2472                                 store_reg_to_var_flt(iptr->dst, d);
2473                                 break;
2474                         }
2475                         break;
2476
2477                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2478                                       /* op1 = type, val.i = field offset             */
2479
2480                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2481                         gen_nullptr_check(s1);
2482                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2483                                 var_to_reg_int(s2, src, REG_ITMP2);
2484                         } else {
2485                                 var_to_reg_flt(s2, src, REG_FTMP2);
2486                         }
2487
2488                         if (!iptr->val.a) {
2489                                 codegen_addpatchref(cd, cd->mcodeptr,
2490                                                                         PATCHER_get_putfield,
2491                                                                         (unresolved_field *) iptr->target, 0);
2492
2493                                 if (opt_showdisassemble) {
2494                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2495                                 }
2496
2497                                 a = 0;
2498
2499                         } else {
2500                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2501                         }
2502
2503                         switch (iptr->op1) {
2504                         case TYPE_INT:
2505                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2506                                 break;
2507                         case TYPE_LNG:
2508                         case TYPE_ADR:
2509                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2510                                 break;
2511                         case TYPE_FLT:
2512                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2513                                 break;
2514                         case TYPE_DBL:
2515                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2516                                 break;
2517                         }
2518                         break;
2519
2520                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2521                                           /* val = value (in current instruction)     */
2522                                           /* op1 = type, val.a = field address (in    */
2523                                           /* following NOP)                           */
2524
2525                         var_to_reg_int(s1, src, REG_ITMP1);
2526                         gen_nullptr_check(s1);
2527
2528                         if (!iptr[1].val.a) {
2529                                 codegen_addpatchref(cd, cd->mcodeptr,
2530                                                                         PATCHER_putfieldconst,
2531                                                                         (unresolved_field *) iptr[1].target, 0);
2532
2533                                 if (opt_showdisassemble) {
2534                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2535                                 }
2536
2537                                 a = 0;
2538
2539                         } else {
2540                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2541                         }
2542
2543                         switch (iptr->op1) {
2544                         case TYPE_INT:
2545                         case TYPE_FLT:
2546                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2547                                 break;
2548                         case TYPE_LNG:
2549                         case TYPE_ADR:
2550                         case TYPE_DBL:
2551                                 /* We can only optimize the move, if the class is resolved.   */
2552                                 /* Otherwise we don't know what to patch.                     */
2553                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2554                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2555                                 } else {
2556                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2557                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2558                                 }
2559                                 break;
2560                         }
2561                         break;
2562
2563
2564                 /* branch operations **************************************************/
2565
2566                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2567
2568                         var_to_reg_int(s1, src, REG_ITMP1);
2569                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2570
2571                         if (iptr->val.a) {
2572                                 codegen_addpatchref(cd, cd->mcodeptr,
2573                                                                         PATCHER_athrow_areturn,
2574                                                                         (unresolved_class *) iptr->val.a, 0);
2575
2576                                 if (opt_showdisassemble) {
2577                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2578                                 }
2579                         }
2580
2581                         M_CALL_IMM(0);                            /* passing exception pc */
2582                         M_POP(REG_ITMP2_XPC);
2583
2584                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
2585                         M_JMP(REG_ITMP3);
2586                         break;
2587
2588                 case ICMD_GOTO:         /* ... ==> ...                                */
2589                                         /* op1 = target JavaVM pc                     */
2590
2591                         M_JMP_IMM(0);
2592                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2593                         break;
2594
2595                 case ICMD_JSR:          /* ... ==> ...                                */
2596                                         /* op1 = target JavaVM pc                     */
2597
2598                         M_CALL_IMM(0);
2599                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2600                         break;
2601
2602                 case ICMD_RET:          /* ... ==> ...                                */
2603                                         /* op1 = local variable                       */
2604
2605                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2606                         var_to_reg_int(s1, var, REG_ITMP1);
2607                         M_JMP(s1);
2608                         break;
2609
2610                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2611                                         /* op1 = target JavaVM pc                     */
2612
2613                         if (src->flags & INMEMORY) {
2614                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2615
2616                         } else {
2617                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2618                         }
2619                         x86_64_jcc(cd, X86_64_CC_E, 0);
2620                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2621                         break;
2622
2623                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2624                                         /* op1 = target JavaVM pc                     */
2625
2626                         if (src->flags & INMEMORY) {
2627                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2628
2629                         } else {
2630                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2631                         }
2632                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2633                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2634                         break;
2635
2636                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2637                                         /* op1 = target JavaVM pc, val.i = constant   */
2638
2639                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2640                         break;
2641
2642                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2643                                         /* op1 = target JavaVM pc, val.i = constant   */
2644
2645                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2646                         break;
2647
2648                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2649                                         /* op1 = target JavaVM pc, val.i = constant   */
2650
2651                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2652                         break;
2653
2654                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2655                                         /* op1 = target JavaVM pc, val.i = constant   */
2656
2657                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2658                         break;
2659
2660                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2661                                         /* op1 = target JavaVM pc, val.i = constant   */
2662
2663                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2664                         break;
2665
2666                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2667                                         /* op1 = target JavaVM pc, val.i = constant   */
2668
2669                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2670                         break;
2671
2672                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2673                                         /* op1 = target JavaVM pc, val.l = constant   */
2674
2675                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2676                         break;
2677
2678                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2679                                         /* op1 = target JavaVM pc, val.l = constant   */
2680
2681                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2682                         break;
2683
2684                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2685                                         /* op1 = target JavaVM pc, val.l = constant   */
2686
2687                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2688                         break;
2689
2690                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2691                                         /* op1 = target JavaVM pc, val.l = constant   */
2692
2693                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2694                         break;
2695
2696                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2697                                         /* op1 = target JavaVM pc, val.l = constant   */
2698
2699                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2700                         break;
2701
2702                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2703                                         /* op1 = target JavaVM pc, val.l = constant   */
2704
2705                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2706                         break;
2707
2708                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2709                                         /* op1 = target JavaVM pc                     */
2710
2711                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2712                         break;
2713
2714                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2715                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2716
2717                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2718                         break;
2719
2720                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2721                                         /* op1 = target JavaVM pc                     */
2722
2723                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2724                         break;
2725
2726                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2727                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2728
2729                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2730                         break;
2731
2732                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2733                                         /* op1 = target JavaVM pc                     */
2734
2735                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2736                         break;
2737
2738                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2739                                     /* op1 = target JavaVM pc                     */
2740
2741                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2742                         break;
2743
2744                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2745                                         /* op1 = target JavaVM pc                     */
2746
2747                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2748                         break;
2749
2750                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2751                                 /* op1 = target JavaVM pc                     */
2752
2753                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2754                         break;
2755
2756                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2757                                         /* op1 = target JavaVM pc                     */
2758
2759                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2760                         break;
2761
2762                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2763                                         /* op1 = target JavaVM pc                     */
2764
2765                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2766                         break;
2767
2768                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2769                                         /* op1 = target JavaVM pc                     */
2770
2771                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2772                         break;
2773
2774                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2775                                     /* op1 = target JavaVM pc                     */
2776
2777                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2778                         break;
2779
2780                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2781
2782                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2783                         break;
2784
2785                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2786                                         /* val.i = constant                           */
2787
2788                         var_to_reg_int(s1, src, REG_ITMP1);
2789                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2790                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2791                                 if (s1 == d) {
2792                                         M_INTMOVE(s1, REG_ITMP1);
2793                                         s1 = REG_ITMP1;
2794                                 }
2795                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2796                         }
2797                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2798                         x86_64_testl_reg_reg(cd, s1, s1);
2799                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2800                         store_reg_to_var_int(iptr->dst, d);
2801                         break;
2802
2803                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2804                                         /* val.i = constant                           */
2805
2806                         var_to_reg_int(s1, src, REG_ITMP1);
2807                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2808                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2809                                 if (s1 == d) {
2810                                         M_INTMOVE(s1, REG_ITMP1);
2811                                         s1 = REG_ITMP1;
2812                                 }
2813                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2814                         }
2815                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2816                         x86_64_testl_reg_reg(cd, s1, s1);
2817                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2818                         store_reg_to_var_int(iptr->dst, d);
2819                         break;
2820
2821                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2822                                         /* val.i = constant                           */
2823
2824                         var_to_reg_int(s1, src, REG_ITMP1);
2825                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2826                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2827                                 if (s1 == d) {
2828                                         M_INTMOVE(s1, REG_ITMP1);
2829                                         s1 = REG_ITMP1;
2830                                 }
2831                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2832                         }
2833                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2834                         x86_64_testl_reg_reg(cd, s1, s1);
2835                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2836                         store_reg_to_var_int(iptr->dst, d);
2837                         break;
2838
2839                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2840                                         /* val.i = constant                           */
2841
2842                         var_to_reg_int(s1, src, REG_ITMP1);
2843                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2844                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2845                                 if (s1 == d) {
2846                                         M_INTMOVE(s1, REG_ITMP1);
2847                                         s1 = REG_ITMP1;
2848                                 }
2849                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2850                         }
2851                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2852                         x86_64_testl_reg_reg(cd, s1, s1);
2853                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2854                         store_reg_to_var_int(iptr->dst, d);
2855                         break;
2856
2857                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2858                                         /* val.i = constant                           */
2859
2860                         var_to_reg_int(s1, src, REG_ITMP1);
2861                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2862                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2863                                 if (s1 == d) {
2864                                         M_INTMOVE(s1, REG_ITMP1);
2865                                         s1 = REG_ITMP1;
2866                                 }
2867                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2868                         }
2869                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2870                         x86_64_testl_reg_reg(cd, s1, s1);
2871                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2872                         store_reg_to_var_int(iptr->dst, d);
2873                         break;
2874
2875                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2876                                         /* val.i = constant                           */
2877
2878                         var_to_reg_int(s1, src, REG_ITMP1);
2879                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2880                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2881                                 if (s1 == d) {
2882                                         M_INTMOVE(s1, REG_ITMP1);
2883                                         s1 = REG_ITMP1;
2884                                 }
2885                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2886                         }
2887                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2888                         x86_64_testl_reg_reg(cd, s1, s1);
2889                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2890                         store_reg_to_var_int(iptr->dst, d);
2891                         break;
2892
2893
2894                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2895                 case ICMD_LRETURN:
2896
2897                         var_to_reg_int(s1, src, REG_RESULT);
2898                         M_INTMOVE(s1, REG_RESULT);
2899                         goto nowperformreturn;
2900
2901                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2902
2903                         var_to_reg_int(s1, src, REG_RESULT);
2904                         M_INTMOVE(s1, REG_RESULT);
2905
2906                         if (iptr->val.a) {
2907                                 codegen_addpatchref(cd, cd->mcodeptr,
2908                                                                         PATCHER_athrow_areturn,
2909                                                                         (unresolved_class *) iptr->val.a, 0);
2910
2911                                 if (opt_showdisassemble) {
2912                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2913                                 }
2914                         }
2915                         goto nowperformreturn;
2916
2917                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2918                 case ICMD_DRETURN:
2919
2920                         var_to_reg_flt(s1, src, REG_FRESULT);
2921                         M_FLTMOVE(s1, REG_FRESULT);
2922                         goto nowperformreturn;
2923
2924                 case ICMD_RETURN:      /* ...  ==> ...                                */
2925
2926 nowperformreturn:
2927                         {
2928                         s4 i, p;
2929
2930                         p = parentargs_base;
2931
2932                         /* call trace function */
2933                         if (runverbose) {
2934                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2935
2936                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2937                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2938
2939                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2940                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2941                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2942                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2943
2944                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2945                                 x86_64_call_reg(cd, REG_ITMP1);
2946
2947                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2948                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2949
2950                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2951                         }
2952
2953 #if defined(USE_THREADS)
2954                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2955                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2956
2957                                 /* we need to save the proper return value */
2958                                 switch (iptr->opc) {
2959                                 case ICMD_IRETURN:
2960                                 case ICMD_ARETURN:
2961                                 case ICMD_LRETURN:
2962                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2963                                         break;
2964                                 case ICMD_FRETURN:
2965                                 case ICMD_DRETURN:
2966                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2967                                         break;
2968                                 }
2969
2970                                 M_MOV_IMM((ptrint) builtin_monitorexit, REG_ITMP1);
2971                                 M_CALL(REG_ITMP1);
2972
2973                                 /* and now restore the proper return value */
2974                                 switch (iptr->opc) {
2975                                 case ICMD_IRETURN:
2976                                 case ICMD_ARETURN:
2977                                 case ICMD_LRETURN:
2978                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2979                                         break;
2980                                 case ICMD_FRETURN:
2981                                 case ICMD_DRETURN:
2982                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2983                                         break;
2984                                 }
2985                         }
2986 #endif
2987
2988                         /* restore saved registers */
2989
2990                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2991                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2992                         }
2993                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2994                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2995                         }
2996
2997                         /* deallocate stack */
2998
2999                         if (parentargs_base)
3000                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
3001
3002                         M_RET;
3003                         }
3004                         break;
3005
3006
3007                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3008                         {
3009                                 s4 i, l, *s4ptr;
3010                                 void **tptr;
3011
3012                                 tptr = (void **) iptr->target;
3013
3014                                 s4ptr = iptr->val.a;
3015                                 l = s4ptr[1];                          /* low     */
3016                                 i = s4ptr[2];                          /* high    */
3017
3018                                 var_to_reg_int(s1, src, REG_ITMP1);
3019                                 M_INTMOVE(s1, REG_ITMP1);
3020                                 if (l != 0) {
3021                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3022                                 }
3023                                 i = i - l + 1;
3024
3025                 /* range check */
3026                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3027                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3028
3029                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3030                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3031
3032                                 /* build jump table top down and use address of lowest entry */
3033
3034                 /* s4ptr += 3 + i; */
3035                                 tptr += i;
3036
3037                                 while (--i >= 0) {
3038                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3039                                         --tptr;
3040                                 }
3041
3042                                 /* length of dataseg after last dseg_addtarget is used by load */
3043
3044                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3045                                 dseg_adddata(cd, cd->mcodeptr);
3046                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3047                                 x86_64_jmp_reg(cd, REG_ITMP1);
3048                         }
3049                         break;
3050
3051
3052                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3053                         {
3054                                 s4 i, l, val, *s4ptr;
3055                                 void **tptr;
3056
3057                                 tptr = (void **) iptr->target;
3058
3059                                 s4ptr = iptr->val.a;
3060                                 l = s4ptr[0];                          /* default  */
3061                                 i = s4ptr[1];                          /* count    */
3062
3063                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3064                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3065                                 while (--i >= 0) {
3066                                         s4ptr += 2;
3067                                         ++tptr;
3068
3069                                         val = s4ptr[0];
3070                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3071                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3072                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3073                                 }
3074
3075                                 x86_64_jmp_imm(cd, 0);
3076
3077                                 tptr = (void **) iptr->target;
3078                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3079                         }
3080                         break;
3081
3082
3083                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3084                                         /* op1 = arg count val.a = builtintable entry */
3085
3086                         bte = iptr->val.a;
3087                         md = bte->md;
3088                         goto gen_method;
3089
3090                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3091                                         /* op1 = arg count, val.a = method pointer    */
3092
3093                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3094                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3095                 case ICMD_INVOKEINTERFACE:
3096
3097                         lm = iptr->val.a;
3098
3099                         if (lm == NULL) {
3100                                 unresolved_method *um = iptr->target;
3101                                 md = um->methodref->parseddesc.md;
3102                         } else {
3103                                 md = lm->parseddesc;
3104                         }
3105
3106 gen_method:
3107                         s3 = md->paramcount;
3108
3109                         MCODECHECK((20 * s3) + 128);
3110
3111                         /* copy arguments to registers or stack location */
3112
3113                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3114                                 if (src->varkind == ARGVAR)
3115                                         continue;
3116                                 if (IS_INT_LNG_TYPE(src->type)) {
3117                                         if (!md->params[s3].inmemory) {
3118                                                 s1 = rd->argintregs[md->params[s3].regoff];
3119                                                 var_to_reg_int(d, src, s1);
3120                                                 M_INTMOVE(d, s1);
3121                                         } else {
3122                                                 var_to_reg_int(d, src, REG_ITMP1);
3123                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3124                                         }
3125
3126                                 } else {
3127                                         if (!md->params[s3].inmemory) {
3128                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3129                                                 var_to_reg_flt(d, src, s1);
3130                                                 M_FLTMOVE(d, s1);
3131                                         } else {
3132                                                 var_to_reg_flt(d, src, REG_FTMP1);
3133                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3134                                         }
3135                                 }
3136                         }
3137
3138                         switch (iptr->opc) {
3139                         case ICMD_BUILTIN:
3140                                 a = (ptrint) bte->fp;
3141                                 d = md->returntype.type;
3142
3143                                 M_MOV_IMM(a, REG_ITMP1);
3144                                 M_CALL(REG_ITMP1);
3145
3146                                 /* if op1 == true, we need to check for an exception */
3147
3148                                 if (iptr->op1 == true) {
3149                                         M_TEST(REG_RESULT);
3150                                         M_BEQ(0);
3151                                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3152                                 }
3153                                 break;
3154
3155                         case ICMD_INVOKESPECIAL:
3156                                 M_TEST(rd->argintregs[0]);
3157                                 M_BEQ(0);
3158                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3159
3160                                 /* first argument contains pointer */
3161 /*                              gen_nullptr_check(rd->argintregs[0]); */
3162
3163                                 /* access memory for hardware nullptr */
3164 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3165
3166                                 /* fall through */
3167
3168                         case ICMD_INVOKESTATIC:
3169                                 if (lm == NULL) {
3170                                         unresolved_method *um = iptr->target;
3171
3172                                         codegen_addpatchref(cd, cd->mcodeptr,
3173                                                                                 PATCHER_invokestatic_special, um, 0);
3174
3175                                         if (opt_showdisassemble) {
3176                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3177                                         }
3178
3179                                         a = 0;
3180                                         d = um->methodref->parseddesc.md->returntype.type;
3181
3182                                 } else {
3183                                         a = (ptrint) lm->stubroutine;
3184                                         d = lm->parseddesc->returntype.type;
3185                                 }
3186
3187                                 M_MOV_IMM(a, REG_ITMP2);
3188                                 M_CALL(REG_ITMP2);
3189                                 break;
3190
3191                         case ICMD_INVOKEVIRTUAL:
3192                                 gen_nullptr_check(rd->argintregs[0]);
3193
3194                                 if (lm == NULL) {
3195                                         unresolved_method *um = iptr->target;
3196
3197                                         codegen_addpatchref(cd, cd->mcodeptr,
3198                                                                                 PATCHER_invokevirtual, um, 0);
3199
3200                                         if (opt_showdisassemble) {
3201                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3202                                         }
3203
3204                                         s1 = 0;
3205                                         d = um->methodref->parseddesc.md->returntype.type;
3206
3207                                 } else {
3208                                         s1 = OFFSET(vftbl_t, table[0]) +
3209                                                 sizeof(methodptr) * lm->vftblindex;
3210                                         d = lm->parseddesc->returntype.type;
3211                                 }
3212
3213                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3214                                                                            OFFSET(java_objectheader, vftbl),
3215                                                                            REG_ITMP2);
3216                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3217                                 M_CALL(REG_ITMP1);
3218                                 break;
3219
3220                         case ICMD_INVOKEINTERFACE:
3221                                 gen_nullptr_check(rd->argintregs[0]);
3222
3223                                 if (lm == NULL) {
3224                                         unresolved_method *um = iptr->target;
3225
3226                                         codegen_addpatchref(cd, cd->mcodeptr,
3227                                                                                 PATCHER_invokeinterface, um, 0);
3228
3229                                         if (opt_showdisassemble) {
3230                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3231                                         }
3232
3233                                         s1 = 0;
3234                                         s2 = 0;
3235                                         d = um->methodref->parseddesc.md->returntype.type;
3236
3237                                 } else {
3238                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3239                                                 sizeof(methodptr) * lm->class->index;
3240
3241                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3242
3243                                         d = lm->parseddesc->returntype.type;
3244                                 }
3245
3246                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3247                                           OFFSET(java_objectheader, vftbl));
3248                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3249                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3250                                 M_CALL(REG_ITMP1);
3251                                 break;
3252                         }
3253
3254                         /* d contains return type */
3255
3256                         if (d != TYPE_VOID) {
3257                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3258                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3259                                         M_INTMOVE(REG_RESULT, s1);
3260                                         store_reg_to_var_int(iptr->dst, s1);
3261                                 } else {
3262                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3263                                         M_FLTMOVE(REG_FRESULT, s1);
3264                                         store_reg_to_var_flt(iptr->dst, s1);
3265                                 }
3266                         }
3267                         break;
3268
3269
3270                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3271
3272                                       /* op1:   0 == array, 1 == class                */
3273                                       /* val.a: (classinfo *) superclass              */
3274
3275                         /*  superclass is an interface:
3276                          *
3277                          *  OK if ((sub == NULL) ||
3278                          *         (sub->vftbl->interfacetablelength > super->index) &&
3279                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3280                          *
3281                          *  superclass is a class:
3282                          *
3283                          *  OK if ((sub == NULL) || (0
3284                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3285                          *         super->vftbl->diffval));
3286                          */
3287
3288                         if (iptr->op1 == 1) {
3289                                 /* object type cast-check */
3290
3291                                 classinfo *super;
3292                                 vftbl_t   *supervftbl;
3293                                 s4         superindex;
3294
3295                                 super = (classinfo *) iptr->val.a;
3296
3297                                 if (!super) {
3298                                         superindex = 0;
3299                                         supervftbl = NULL;
3300
3301                                 } else {
3302                                         superindex = super->index;
3303                                         supervftbl = super->vftbl;
3304                                 }
3305
3306 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3307                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3308 #endif
3309                                 var_to_reg_int(s1, src, REG_ITMP1);
3310
3311                                 /* calculate interface checkcast code size */
3312
3313                                 s2 = 3; /* mov_membase_reg */
3314                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3315
3316                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3317                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3318                                         3 /* test */ + 6 /* jcc */;
3319
3320                                 if (!super)
3321                                         s2 += (opt_showdisassemble ? 5 : 0);
3322
3323                                 /* calculate class checkcast code size */
3324
3325                                 s3 = 3; /* mov_membase_reg */
3326                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3327                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3328
3329 #if 0
3330                                 if (s1 != REG_ITMP1) {
3331                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3332                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3333                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3334                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3335                                         a += 3;    /* sub */
3336
3337                                 } else
3338 #endif
3339                                         {
3340                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3341                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3342                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3343                                         }
3344
3345                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3346
3347                                 if (!super)
3348                                         s3 += (opt_showdisassemble ? 5 : 0);
3349
3350                                 /* if class is not resolved, check which code to call */
3351
3352                                 if (!super) {
3353                                         M_TEST(s1);
3354                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3355
3356                                         codegen_addpatchref(cd, cd->mcodeptr,
3357                                                                                 PATCHER_checkcast_instanceof_flags,
3358                                                                                 (constant_classref *) iptr->target, 0);
3359
3360                                         if (opt_showdisassemble) {
3361                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3362                                         }
3363
3364                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3365                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3366                                         M_BEQ(s2 + 5);
3367                                 }
3368
3369                                 /* interface checkcast code */
3370
3371                                 if (!super || (super->flags & ACC_INTERFACE)) {
3372                                         if (super) {
3373                                                 M_TEST(s1);
3374                                                 M_BEQ(s2);
3375                                         }
3376
3377                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3378
3379                                         if (!super) {
3380                                                 codegen_addpatchref(cd, cd->mcodeptr,
3381                                                                                         PATCHER_checkcast_instanceof_interface,
3382                                                                                         (constant_classref *) iptr->target, 0);
3383
3384                                                 if (opt_showdisassemble) {
3385                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3386                                                 }
3387                                         }
3388
3389                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3390                                                                                           OFFSET(vftbl_t, interfacetablelength),
3391                                                                                           REG_ITMP3);
3392                                         /* XXX TWISTI: should this be int arithmetic? */
3393                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3394                                         M_TEST(REG_ITMP3);
3395                                         M_BLE(0);
3396                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3397                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3398                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3399                                                                                          superindex * sizeof(methodptr*),
3400                                                                                          REG_ITMP3);
3401                                         M_TEST(REG_ITMP3);
3402                                         M_BEQ(0);
3403                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3404
3405                                         if (!super)
3406                                                 M_JMP_IMM(s3);
3407                                 }
3408
3409                                 /* class checkcast code */
3410
3411                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3412                                         if (super) {
3413                                                 M_TEST(s1);
3414                                                 M_BEQ(s3);
3415                                         }
3416
3417                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3418
3419                                         if (!super) {
3420                                                 codegen_addpatchref(cd, cd->mcodeptr,
3421                                                                                         PATCHER_checkcast_class,
3422                                                                                         (constant_classref *) iptr->target,
3423                                                                                         0);
3424
3425                                                 if (opt_showdisassemble) {
3426                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3427                                                 }
3428                                         }
3429
3430                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3431 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3432                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3433 #endif
3434                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3435                                                                                           OFFSET(vftbl_t, baseval),
3436                                                                                           REG_ITMP2);
3437                                         /*                                      if (s1 != REG_ITMP1) { */
3438                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3439                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3440                                         /*                                                                                              REG_ITMP1); */
3441                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3442                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3443                                         /*                                                                                              REG_ITMP3); */
3444                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3445                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3446                                         /*  #endif */
3447                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3448
3449                                         /*                                      } else { */
3450                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3451                                                                                           OFFSET(vftbl_t, baseval),
3452                                                                                           REG_ITMP3);
3453                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3454                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3455                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3456                                         /*                                      } */
3457 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3458                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3459 #endif
3460                                         M_CMP(REG_ITMP3, REG_ITMP2);
3461                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3462                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3463                                 }
3464                                 d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3465
3466                         } else {
3467                                 /* array type cast-check */
3468
3469                                 var_to_reg_int(s1, src, REG_ITMP1);
3470                                 M_INTMOVE(s1, rd->argintregs[0]);
3471
3472                                 if (iptr->val.a == NULL) {
3473                                         codegen_addpatchref(cd, cd->mcodeptr,
3474                                                                                 PATCHER_builtin_arraycheckcast,
3475                                                                                 (constant_classref *) iptr->target, 0);
3476
3477                                         if (opt_showdisassemble) {
3478                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3479                                         }
3480                                 }
3481
3482                                 M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3483                                 M_MOV_IMM((ptrint) BUILTIN_arraycheckcast, REG_ITMP1);
3484                                 M_CALL(REG_ITMP1);
3485                                 M_TEST(REG_RESULT);
3486                                 M_BEQ(0);
3487                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3488
3489                                 var_to_reg_int(s1, src, REG_ITMP1);
3490                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3491                         }
3492                         M_INTMOVE(s1, d);
3493                         store_reg_to_var_int(iptr->dst, d);
3494                         break;
3495
3496                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3497
3498                                       /* op1:   0 == array, 1 == class                */
3499                                       /* val.a: (classinfo *) superclass              */
3500
3501                         /*  superclass is an interface:
3502                          *
3503                          *  return (sub != NULL) &&
3504                          *         (sub->vftbl->interfacetablelength > super->index) &&
3505                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3506                          *
3507                          *  superclass is a class:
3508                          *
3509                          *  return ((sub != NULL) && (0
3510                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3511                          *          super->vftbl->diffvall));
3512                          */
3513
3514                         {
3515                         classinfo *super;
3516                         vftbl_t   *supervftbl;
3517                         s4         superindex;
3518
3519                         super = (classinfo *) iptr->val.a;
3520
3521                         if (!super) {
3522                                 superindex = 0;
3523                                 supervftbl = NULL;
3524
3525                         } else {
3526                                 superindex = super->index;
3527                                 supervftbl = super->vftbl;
3528                         }
3529
3530 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3531             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3532 #endif
3533
3534                         var_to_reg_int(s1, src, REG_ITMP1);
3535                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3536                         if (s1 == d) {
3537                                 M_INTMOVE(s1, REG_ITMP1);
3538                                 s1 = REG_ITMP1;
3539                         }
3540
3541                         /* calculate interface instanceof code size */
3542
3543                         s2 = 3; /* mov_membase_reg */
3544                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3545                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3546                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3547                                 3 /* test */ + 4 /* setcc */;
3548
3549                         if (!super)
3550                                 s2 += (opt_showdisassemble ? 5 : 0);
3551
3552                         /* calculate class instanceof code size */
3553
3554                         s3 = 3; /* mov_membase_reg */
3555                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3556                         s3 += 10; /* mov_imm_reg */
3557                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3558                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3559                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3560                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3561                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3562                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3563                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3564
3565                         if (!super)
3566                                 s3 += (opt_showdisassemble ? 5 : 0);
3567
3568                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3569
3570                         /* if class is not resolved, check which code to call */
3571
3572                         if (!super) {
3573                                 x86_64_test_reg_reg(cd, s1, s1);
3574                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3575                                                                                          7 + 6 + s2 + 5 + s3));
3576
3577                                 codegen_addpatchref(cd, cd->mcodeptr,
3578                                                                         PATCHER_checkcast_instanceof_flags,
3579                                                                         (constant_classref *) iptr->target, 0);
3580
3581                                 if (opt_showdisassemble) {
3582                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3583                                 }
3584
3585                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3586                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3587                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3588                         }
3589
3590                         /* interface instanceof code */
3591
3592                         if (!super || (super->flags & ACC_INTERFACE)) {
3593                                 if (super) {
3594                                         x86_64_test_reg_reg(cd, s1, s1);
3595                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3596                                 }
3597
3598                                 x86_64_mov_membase_reg(cd, s1,
3599                                                                            OFFSET(java_objectheader, vftbl),
3600                                                                            REG_ITMP1);
3601                                 if (!super) {
3602                                         codegen_addpatchref(cd, cd->mcodeptr,
3603                                                                                 PATCHER_checkcast_instanceof_interface,
3604                                                                                 (constant_classref *) iptr->target, 0);
3605
3606                                         if (opt_showdisassemble) {
3607                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3608                                         }
3609                                 }
3610
3611                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3612                                                                                   OFFSET(vftbl_t, interfacetablelength),
3613                                                                                   REG_ITMP3);
3614                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3615                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3616
3617                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3618
3619                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3620                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3621                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3622                                                                                  superindex * sizeof(methodptr*),
3623                                                                                  REG_ITMP1);
3624                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3625                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3626
3627                                 if (!super)
3628                                         x86_64_jmp_imm(cd, s3);
3629                         }
3630
3631                         /* class instanceof code */
3632
3633                         if (!super || !(super->flags & ACC_INTERFACE)) {
3634                                 if (super) {
3635                                         x86_64_test_reg_reg(cd, s1, s1);
3636                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3637                                 }
3638
3639                                 x86_64_mov_membase_reg(cd, s1,
3640                                                                            OFFSET(java_objectheader, vftbl),
3641                                                                            REG_ITMP1);
3642
3643                                 if (!super) {
3644                                         codegen_addpatchref(cd, cd->mcodeptr,
3645                                                                                 PATCHER_instanceof_class,
3646                                                                                 (constant_classref *) iptr->target, 0);
3647
3648                                         if (opt_showdisassemble) {
3649                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3650                                         }
3651                                 }
3652
3653                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3654 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3655                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3656 #endif
3657                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3658                                                                                 OFFSET(vftbl_t, baseval),
3659                                                                                 REG_ITMP1);
3660                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3661                                                                                 OFFSET(vftbl_t, diffval),
3662                                                                                 REG_ITMP3);
3663                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3664                                                                                 OFFSET(vftbl_t, baseval),
3665                                                                                 REG_ITMP2);
3666 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3667                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3668 #endif
3669                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3670                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3671                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3672                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3673                         }
3674                         store_reg_to_var_int(iptr->dst, d);
3675                         }
3676                         break;
3677
3678                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3679                                          /* op1 = dimension, val.a = array descriptor */
3680
3681                         /* check for negative sizes and copy sizes to stack if necessary  */
3682
3683                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3684
3685                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3686                                 /* copy SAVEDVAR sizes to stack */
3687
3688                                 if (src->varkind != ARGVAR) {
3689                                         var_to_reg_int(s2, src, REG_ITMP1);
3690                                         M_LST(s2, REG_SP, s1 * 8);
3691                                 }
3692                         }
3693
3694                         /* is a patcher function set? */
3695
3696                         if (iptr->target) {
3697                                 codegen_addpatchref(cd, cd->mcodeptr,
3698                                                                         (functionptr) (ptrint) iptr->target,
3699                                                                         iptr->val.a, 0);
3700
3701                                 if (opt_showdisassemble) {
3702                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3703                                 }
3704
3705                                 a = 0;
3706
3707                         } else {
3708                                 a = (ptrint) iptr->val.a;
3709                         }
3710
3711                         /* a0 = dimension count */
3712
3713                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3714
3715                         /* a1 = arrayvftbl */
3716
3717                         M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3718
3719                         /* a2 = pointer to dimensions = stack pointer */
3720
3721                         M_MOV(REG_SP, rd->argintregs[2]);
3722
3723                         M_MOV_IMM((ptrint) BUILTIN_multianewarray, REG_ITMP1);
3724                         M_CALL(REG_ITMP1);
3725
3726                         /* check for exception before result assignment */
3727
3728                         M_TEST(REG_RESULT);
3729                         M_BEQ(0);
3730                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3731
3732                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3733                         M_INTMOVE(REG_RESULT, s1);
3734                         store_reg_to_var_int(iptr->dst, s1);
3735                         break;
3736
3737                 default:
3738                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3739                         return false;
3740         } /* switch */
3741
3742         } /* for instruction */
3743
3744         /* copy values to interface registers */
3745
3746         src = bptr->outstack;
3747         len = bptr->outdepth;
3748         MCODECHECK(512);
3749 #ifdef LSRA
3750         if (!opt_lsra)
3751 #endif
3752         while (src) {
3753                 len--;
3754                 if ((src->varkind != STACKVAR)) {
3755                         s2 = src->type;
3756                         if (IS_FLT_DBL_TYPE(s2)) {
3757                                 var_to_reg_flt(s1, src, REG_FTMP1);
3758                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3759                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3760
3761                                 } else {
3762                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3763                                 }
3764
3765                         } else {
3766                                 var_to_reg_int(s1, src, REG_ITMP1);
3767                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3768                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3769
3770                                 } else {
3771                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3772                                 }
3773                         }
3774                 }
3775                 src = src->prev;
3776         }
3777
3778         /* At the end of a basic block we may have to append some nops,
3779            because the patcher stub calling code might be longer than the
3780            actual instruction. So codepatching does not change the
3781            following block unintentionally. */
3782
3783         if (cd->mcodeptr < cd->lastmcodeptr) {
3784                 while (cd->mcodeptr < cd->lastmcodeptr) {
3785                         M_NOP;
3786                 }
3787         }
3788
3789         } /* if (bptr -> flags >= BBREACHED) */
3790         } /* for basic block */
3791
3792         codegen_createlinenumbertable(cd);
3793
3794         {
3795
3796         u1        *xcodeptr;
3797         branchref *bref;
3798
3799         /* generate ArithmeticException stubs */
3800
3801         xcodeptr = NULL;
3802
3803         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3804                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3805                                   bref->branchpos,
3806                                                   cd->mcodeptr - cd->mcodebase);
3807
3808                 MCODECHECK(512);
3809
3810                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3811                 dseg_adddata(cd, cd->mcodeptr);
3812                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3813
3814                 if (xcodeptr != NULL) {
3815                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3816
3817                 } else {
3818                         xcodeptr = cd->mcodeptr;
3819
3820                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3821                         M_MOV(REG_SP, rd->argintregs[1]);
3822                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3823                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3824
3825                         M_ASUB_IMM(2 * 8, REG_SP);
3826                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3827
3828                         M_MOV_IMM((ptrint) stacktrace_inline_arithmeticexception,
3829                                           REG_ITMP3);
3830                         M_CALL(REG_ITMP3);
3831
3832                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3833                         M_AADD_IMM(2 * 8, REG_SP);
3834
3835                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3836                         M_JMP(REG_ITMP3);
3837                 }
3838         }
3839
3840         /* generate ArrayIndexOutOfBoundsException stubs */
3841
3842         xcodeptr = NULL;
3843
3844         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3845                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3846                                   bref->branchpos,
3847                                                   cd->mcodeptr - cd->mcodebase);
3848
3849                 MCODECHECK(512);
3850
3851                 /* move index register into REG_ITMP1 */
3852
3853                 M_MOV(bref->reg, REG_ITMP1);                              /* 3 bytes  */
3854
3855                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3856                 dseg_adddata(cd, cd->mcodeptr);
3857                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3858
3859                 if (xcodeptr != NULL) {
3860                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3861
3862                 } else {
3863                         xcodeptr = cd->mcodeptr;
3864
3865                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3866                         M_MOV(REG_SP, rd->argintregs[1]);
3867                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3868                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3869                         M_MOV(REG_ITMP1, rd->argintregs[4]);
3870
3871                         M_ASUB_IMM(2 * 8, REG_SP);
3872                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3873
3874                         M_MOV_IMM((ptrint) stacktrace_inline_arrayindexoutofboundsexception,
3875                                           REG_ITMP3);
3876                         M_CALL(REG_ITMP3);
3877
3878                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3879                         M_AADD_IMM(2 * 8, REG_SP);
3880
3881                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3882                         M_JMP(REG_ITMP3);
3883                 }
3884         }
3885
3886         /* generate ArrayStoreException stubs */
3887
3888         xcodeptr = NULL;
3889
3890         for (bref = cd->xstorerefs; bref != NULL; bref = bref->next) {
3891                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3892                                   bref->branchpos,
3893                                                   cd->mcodeptr - cd->mcodebase);
3894
3895                 MCODECHECK(512);
3896
3897                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3898                 dseg_adddata(cd, cd->mcodeptr);
3899                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3900
3901                 if (xcodeptr != NULL) {
3902                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3903
3904                 } else {
3905                         xcodeptr = cd->mcodeptr;
3906
3907                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3908                         M_MOV(REG_SP, rd->argintregs[1]);
3909                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3910                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3911
3912                         M_ASUB_IMM(2 * 8, REG_SP);
3913                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3914
3915                         M_MOV_IMM((ptrint) stacktrace_inline_arraystoreexception,
3916                                           REG_ITMP3);
3917                         M_CALL(REG_ITMP3);
3918
3919                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3920                         M_AADD_IMM(2 * 8, REG_SP);
3921
3922                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3923                         M_JMP(REG_ITMP3);
3924                 }
3925         }
3926
3927         /* generate ClassCastException stubs */
3928
3929         xcodeptr = NULL;
3930
3931         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3932                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3933                                   bref->branchpos,
3934                                                   cd->mcodeptr - cd->mcodebase);
3935
3936                 MCODECHECK(512);
3937
3938                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3939                 dseg_adddata(cd, cd->mcodeptr);
3940                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3941
3942                 if (xcodeptr != NULL) {
3943                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3944
3945                 } else {
3946                         xcodeptr = cd->mcodeptr;
3947
3948                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3949                         M_MOV(REG_SP, rd->argintregs[1]);
3950                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3951                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3952
3953                         M_ASUB_IMM(2 * 8, REG_SP);
3954                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3955
3956                         M_MOV_IMM((ptrint) stacktrace_inline_classcastexception, REG_ITMP3);
3957                         M_CALL(REG_ITMP3);
3958
3959                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3960                         M_AADD_IMM(2 * 8, REG_SP);
3961
3962                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3963                         M_JMP(REG_ITMP3);
3964                 }
3965         }
3966
3967         /* generate NullpointerException stubs */
3968
3969         xcodeptr = NULL;
3970
3971         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3972                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3973                                   bref->branchpos,
3974                                                   cd->mcodeptr - cd->mcodebase);
3975
3976                 MCODECHECK(512);
3977
3978                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3979                 dseg_adddata(cd, cd->mcodeptr);
3980                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3981
3982                 if (xcodeptr != NULL) {
3983                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3984
3985                 } else {
3986                         xcodeptr = cd->mcodeptr;
3987
3988                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3989                         M_MOV(REG_SP, rd->argintregs[1]);
3990                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3991                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3992
3993                         M_ASUB_IMM(2 * 8, REG_SP);
3994                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3995
3996                         M_MOV_IMM((ptrint) stacktrace_inline_nullpointerexception,
3997                                           REG_ITMP3);
3998                         M_CALL(REG_ITMP3);
3999
4000                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4001                         M_AADD_IMM(2 * 8, REG_SP);
4002
4003                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4004                         M_JMP(REG_ITMP3);
4005                 }
4006         }
4007
4008         /* generate exception check stubs */
4009
4010         xcodeptr = NULL;
4011
4012         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
4013                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4014                                   bref->branchpos,
4015                                                   cd->mcodeptr - cd->mcodebase);
4016
4017                 MCODECHECK(512);
4018
4019                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4020                 dseg_adddata(cd, cd->mcodeptr);
4021                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4022
4023                 if (xcodeptr != NULL) {
4024                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4025
4026                 } else {
4027                         xcodeptr = cd->mcodeptr;
4028
4029                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4030                         M_MOV(REG_SP, rd->argintregs[1]);
4031                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4032                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4033
4034                         M_ASUB_IMM(2 * 8, REG_SP);
4035                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4036
4037                         M_MOV_IMM((ptrint) stacktrace_inline_fillInStackTrace, REG_ITMP3);
4038                         M_CALL(REG_ITMP3);
4039
4040                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4041                         M_AADD_IMM(2 * 8, REG_SP);
4042
4043                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4044                         M_JMP(REG_ITMP3);
4045                 }
4046         }
4047
4048         /* generate code patching stub call code */
4049
4050         {
4051                 patchref    *pref;
4052                 codegendata *tmpcd;
4053                 ptrint       mcode;
4054
4055                 tmpcd = DNEW(codegendata);
4056
4057                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4058                         /* check size of code segment */
4059
4060                         MCODECHECK(512);
4061
4062                         /* Get machine code which is patched back in later. A             */
4063                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4064
4065                         xcodeptr = cd->mcodebase + pref->branchpos;
4066                         mcode = *((ptrint *) xcodeptr);
4067
4068                         /* patch in `call rel32' to call the following code               */
4069
4070                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4071                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4072
4073                         /* move pointer to java_objectheader onto stack */
4074
4075 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4076                         /* create a virtual java_objectheader */
4077
4078                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4079                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4080
4081                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4082                         M_PUSH(REG_ITMP3);
4083 #else
4084                         M_PUSH_IMM(0);
4085 #endif
4086
4087                         /* move machine code bytes and classinfo pointer into registers */
4088
4089                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4090                         M_PUSH(REG_ITMP3);
4091                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4092                         M_PUSH(REG_ITMP3);
4093                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4094                         M_PUSH(REG_ITMP3);
4095
4096                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4097                         M_PUSH(REG_ITMP3);
4098
4099                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4100                         M_JMP(REG_ITMP3);
4101                 }
4102         }
4103         }
4104
4105         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4106
4107         /* everything's ok */
4108
4109         return true;
4110 }
4111
4112
4113 /* createcompilerstub **********************************************************
4114
4115    Creates a stub routine which calls the compiler.
4116
4117 *******************************************************************************/
4118
4119 #define COMPILERSTUB_SIZE    23
4120
4121 functionptr createcompilerstub(methodinfo *m)
4122 {
4123         u1          *s;                     /* memory to hold the stub            */
4124         codegendata *cd;
4125         s4           dumpsize;
4126
4127         s = CNEW(u1, COMPILERSTUB_SIZE);
4128
4129         /* mark start of dump memory area */
4130
4131         dumpsize = dump_size();
4132
4133         cd = DNEW(codegendata);
4134         cd->mcodeptr = s;
4135
4136         /* code for the stub */
4137
4138         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1); /* pass method to compiler */
4139         x86_64_mov_imm_reg(cd, (ptrint) asm_call_jit_compiler, REG_ITMP3);
4140         x86_64_jmp_reg(cd, REG_ITMP3);
4141
4142 #if defined(STATISTICS)
4143         if (opt_stat)
4144                 count_cstub_len += COMPILERSTUB_SIZE;
4145 #endif
4146
4147         /* release dump area */
4148
4149         dump_release(dumpsize);
4150
4151         return (functionptr) (ptrint) s;
4152 }
4153
4154
4155 /* createnativestub ************************************************************
4156
4157    Creates a stub routine which calls a native method.
4158
4159 *******************************************************************************/
4160
4161 functionptr createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4162                                                          registerdata *rd, methoddesc *nmd)
4163 {
4164         methoddesc *md;
4165         s4          stackframesize;         /* size of stackframe if needed       */
4166         s4          nativeparams;
4167         s4          i, j;                   /* count variables                    */
4168         s4          t;
4169         s4          s1, s2;
4170
4171         /* initialize variables */
4172
4173         md = m->parseddesc;
4174         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4175
4176
4177         /* calculate stack frame size */
4178
4179         stackframesize =
4180                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4181                 sizeof(localref_table) / SIZEOF_VOID_P +
4182                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4183                 nmd->memuse;
4184
4185         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4186                 stackframesize++;
4187
4188
4189         /* create method header */
4190
4191         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4192         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4193         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4194         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4195         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4196         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4197         (void) dseg_addlinenumbertablesize(cd);
4198         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4199
4200
4201         /* initialize mcode variables */
4202
4203         cd->mcodeptr = (u1 *) cd->mcodebase;
4204         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4205
4206
4207         /* generate stub code */
4208
4209         M_ASUB_IMM(stackframesize * 8, REG_SP);
4210
4211         if (runverbose) {
4212                 /* save integer and float argument registers */
4213
4214                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4215                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4216                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4217
4218                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4219                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4220                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4221
4222                 /* show integer hex code for float arguments */
4223
4224                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++) {
4225                         /* if the paramtype is a float, we have to right shift all        */
4226                         /* following integer registers                                    */
4227
4228                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4229                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4230                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4231
4232                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4233                                 j++;
4234                         }
4235                 }
4236
4237                 x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1);
4238                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
4239                 x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
4240                 x86_64_call_reg(cd, REG_ITMP1);
4241
4242                 /* restore integer and float argument registers */
4243
4244                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4245                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4246                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4247
4248                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4249                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4250                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4251         }
4252
4253
4254         /* get function address (this must happen before the stackframeinfo) */
4255
4256 #if !defined(ENABLE_STATICVM)
4257         if (f == NULL) {
4258                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4259
4260                 if (opt_showdisassemble) {
4261                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4262                 }
4263         }
4264 #endif
4265
4266         M_MOV_IMM((ptrint) f, REG_ITMP3);
4267
4268
4269         /* save integer and float argument registers */
4270
4271         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4272                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4273                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4274
4275         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4276                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4277                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4278
4279         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4280
4281         /* create dynamic stack info */
4282
4283         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4284         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4285         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4286         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4287         M_MOV_IMM((ptrint) codegen_start_native_call, REG_ITMP1);
4288         M_CALL(REG_ITMP1);
4289
4290         STATS({
4291                 x86_64_mov_imm_reg(cd, (ptrint) nativeinvokation, REG_ITMP1);
4292                 x86_64_call_reg(cd, REG_ITMP1);
4293         })
4294
4295         /* restore integer and float argument registers */
4296
4297         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4298                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4299                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4300
4301         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4302                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4303                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4304
4305         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4306
4307
4308         /* copy or spill arguments to new locations */
4309
4310         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4311                 t = md->paramtypes[i].type;
4312
4313                 if (IS_INT_LNG_TYPE(t)) {
4314                         if (!md->params[i].inmemory) {
4315                                 s1 = rd->argintregs[md->params[i].regoff];
4316
4317                                 if (!nmd->params[j].inmemory) {
4318                                         s2 = rd->argintregs[nmd->params[j].regoff];
4319                                         M_INTMOVE(s1, s2);
4320
4321                                 } else {
4322                                         s2 = nmd->params[j].regoff;
4323                                         M_LST(s1, REG_SP, s2 * 8);
4324                                 }
4325
4326                         } else {
4327                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4328                                 s2 = nmd->params[j].regoff;
4329                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4330                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4331                         }
4332
4333                 } else {
4334                         /* We only copy spilled float arguments, as the float argument    */
4335                         /* registers keep unchanged.                                      */
4336
4337                         if (md->params[i].inmemory) {
4338                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4339                                 s2 = nmd->params[j].regoff;
4340                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4341                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4342                         }
4343                 }
4344         }
4345
4346         /* put class into second argument register */
4347
4348         if (m->flags & ACC_STATIC)
4349                 M_MOV_IMM((ptrint) m->class, rd->argintregs[1]);
4350
4351         /* put env into first argument register */
4352
4353         M_MOV_IMM((ptrint) &env, rd->argintregs[0]);
4354
4355         /* do the native function call */
4356
4357         M_CALL(REG_ITMP3);
4358
4359         /* save return value */
4360
4361         if (IS_INT_LNG_TYPE(md->returntype.type))
4362                 M_LST(REG_RESULT, REG_SP, 0 * 8);
4363         else
4364                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
4365
4366         /* remove native stackframe info */
4367
4368         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4369         M_MOV_IMM((ptrint) codegen_finish_native_call, REG_ITMP1);
4370         M_CALL(REG_ITMP1);
4371
4372         /* generate call trace */
4373
4374         if (runverbose) {
4375                 /* just restore the value we need, don't care about the other */
4376
4377                 if (IS_INT_LNG_TYPE(md->returntype.type))
4378                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4379                 else
4380                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4381
4382                 M_MOV_IMM((ptrint) m, rd->argintregs[0]);
4383                 M_MOV(REG_RESULT, rd->argintregs[1]);
4384                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4385                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4386
4387                 M_MOV_IMM((ptrint) builtin_displaymethodstop, REG_ITMP1);
4388                 M_CALL(REG_ITMP1);
4389         }
4390
4391         /* check for exception */
4392
4393 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4394         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4395         M_CALL(REG_ITMP3);
4396 #else
4397         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_RESULT);
4398 #endif
4399         M_ALD(REG_ITMP3, REG_RESULT, 0);
4400
4401
4402         /* restore return value */
4403
4404         if (IS_INT_LNG_TYPE(md->returntype.type))
4405                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4406         else
4407                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4408
4409         /* test for exception */
4410
4411         M_TEST(REG_ITMP3);
4412         M_BNE(7 + 1);
4413
4414         /* remove stackframe */
4415
4416         M_AADD_IMM(stackframesize * 8, REG_SP);
4417         M_RET;
4418
4419
4420         /* handle exception */
4421
4422 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4423         M_LST(REG_ITMP3, REG_SP, 0 * 8);
4424         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4425         M_CALL(REG_ITMP3);
4426         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4427         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4428 #else
4429         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4430         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_ITMP3);
4431         M_XOR(REG_ITMP2, REG_ITMP2);
4432         M_AST(REG_ITMP2, REG_ITMP3, 0);                /* clear exception pointer */
4433 #endif
4434
4435         /* remove stackframe */
4436
4437         M_AADD_IMM(stackframesize * 8, REG_SP);
4438
4439         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4440         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4441
4442         M_MOV_IMM((ptrint) asm_handle_nat_exception, REG_ITMP3);
4443         M_JMP(REG_ITMP3);
4444
4445
4446         /* process patcher calls **************************************************/
4447
4448         {
4449                 u1          *xcodeptr;
4450                 patchref    *pref;
4451                 codegendata *tmpcd;
4452                 ptrint       mcode;
4453 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4454                 s4           disp;
4455 #endif
4456
4457                 tmpcd = DNEW(codegendata);
4458
4459                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4460                         /* Get machine code which is patched back in later. A             */
4461                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4462
4463                         xcodeptr = cd->mcodebase + pref->branchpos;
4464                         mcode = *((ptrint *) xcodeptr);
4465
4466                         /* patch in `call rel32' to call the following code               */
4467
4468                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4469                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4470
4471                         /* move pointer to java_objectheader onto stack */
4472
4473 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4474                         /* create a virtual java_objectheader */
4475
4476                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4477                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4478
4479                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4480                         M_PUSH(REG_ITMP3);
4481 #else
4482                         M_PUSH_IMM(0);
4483 #endif
4484
4485                         /* move machine code bytes and classinfo pointer into registers */
4486
4487                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4488                         M_PUSH(REG_ITMP3);
4489                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4490                         M_PUSH(REG_ITMP3);
4491                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4492                         M_PUSH(REG_ITMP3);
4493
4494                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4495                         M_PUSH(REG_ITMP3);
4496
4497                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4498                         M_JMP(REG_ITMP3);
4499                 }
4500         }
4501
4502         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4503
4504         return m->entrypoint;
4505 }
4506
4507
4508 /*
4509  * These are local overrides for various environment variables in Emacs.
4510  * Please do not remove this and leave it at the end of the file, where
4511  * Emacs will automagically detect them.
4512  * ---------------------------------------------------------------------
4513  * Local variables:
4514  * mode: c
4515  * indent-tabs-mode: t
4516  * c-basic-offset: 4
4517  * tab-width: 4
4518  * End:
4519  */