src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31
  32    $Id: codegen.c 3621 2005-11-07 18:48:16Z twisti $
  33
  34 */
  35
  36
  37 #include <stdio.h>
  38
  39 #include "config.h"
  40 #include "vm/types.h"
  41
  42 #include "md.h"
  43 #include "md-abi.h"
  44 #include "md-abi.inc"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emitfuncs.h"
  49
  50 #include "cacao/cacao.h"
  51 #include "native/native.h"
  52 #include "vm/global.h"
  53 #include "vm/builtin.h"
  54 #include "vm/loader.h"
  55 #include "vm/statistics.h"
  56 #include "vm/stringlocal.h"
  57 #include "vm/tables.h"
  58 #include "vm/jit/asmpart.h"
  59 #include "vm/jit/codegen.inc"
  60 #include "vm/jit/jit.h"
  61
  62 #if defined(LSRA)
  63 # include "vm/jit/lsra.inc"
  64 #endif
  65
  66 #include "vm/jit/methodheader.h"
  67 #include "vm/jit/parse.h"
  68 #include "vm/jit/patcher.h"
  69 #include "vm/jit/reg.h"
  70 #include "vm/jit/reg.inc"
  71
  72
  73 /* codegen *********************************************************************
  74
  75    Generates machine code.
  76
  77 *******************************************************************************/
  78
  79 void codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  80 {
  81         s4                  len, s1, s2, s3, d, disp;
  82         u2                  currentline;
  83         ptrint              a;
  84         s4                  parentargs_base;
  85         stackptr            src;
  86         varinfo            *var;
  87         basicblock         *bptr;
  88         instruction        *iptr;
  89         exceptiontable     *ex;
  90         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  91         builtintable_entry *bte;
  92         methoddesc         *md;
  93
  94         /* prevent compiler warnings */
  95
  96         d = 0;
  97         lm = NULL;
  98         bte = NULL;
  99
 100         {
 101         s4 i, p, t, l;
 102         s4 savedregs_num;
 103
 104         savedregs_num = 0;
 105
 106         /* space to save used callee saved registers */
 107
 108         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 109         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 110
 111         parentargs_base = rd->memuse + savedregs_num;
 112
 113 #if defined(USE_THREADS)
 114         /* space to save argument of monitor_enter */
 115
 116         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 117                 parentargs_base++;
 118 #endif
 119
 120     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 121         /* code e.g. libc or jni (alignment problems with movaps).                */
 122
 123         if (!m->isleafmethod || runverbose)
 124                 parentargs_base |= 0x1;
 125
 126         /* create method header */
 127
 128         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 129         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 130
 131 #if defined(USE_THREADS)
 132         /* IsSync contains the offset relative to the stack pointer for the
 133            argument of monitor_exit used in the exception handler. Since the
 134            offset could be zero and give a wrong meaning of the flag it is
 135            offset by one.
 136         */
 137
 138         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 139                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 140         else
 141 #endif
 142                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 143
 144         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 145         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 146         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 147
 148         (void) dseg_addlinenumbertablesize(cd);
 149
 150         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 151
 152         /* create exception table */
 153
 154         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 155                 dseg_addtarget(cd, ex->start);
 156                 dseg_addtarget(cd, ex->end);
 157                 dseg_addtarget(cd, ex->handler);
 158                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 159         }
 160
 161         /* initialize mcode variables */
 162
 163         cd->mcodeptr = (u1 *) cd->mcodebase;
 164         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 165         MCODECHECK(512);
 166
 167         /* initialize the last patcher pointer */
 168
 169         cd->lastmcodeptr = cd->mcodeptr;
 170
 171         /* create stack frame (if necessary) */
 172
 173         if (parentargs_base)
 174                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 175
 176         /* save used callee saved registers */
 177
 178         p = parentargs_base;
 179         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 180                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 181         }
 182         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 183                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 184         }
 185
 186         /* take arguments out of register or stack frame */
 187
 188         md = m->parseddesc;
 189
 190         for (p = 0, l = 0; p < md->paramcount; p++) {
 191                 t = md->paramtypes[p].type;
 192                 var = &(rd->locals[l][t]);
 193                 l++;
 194                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 195                         l++;
 196                 if (var->type < 0)
 197                         continue;
 198                 s1 = md->params[p].regoff;
 199                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 200                         s2 = rd->argintregs[s1];
 201                         if (!md->params[p].inmemory) {           /* register arguments    */
 202                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 203                                         M_INTMOVE(s2, var->regoff);
 204
 205                                 } else {                             /* reg arg -> spilled    */
 206                                     M_LST(s2, REG_SP, var->regoff * 8);
 207                                 }
 208
 209                         } else {                                 /* stack arguments       */
 210                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 211                                         /* + 8 for return address */
 212                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 213
 214                                 } else {                             /* stack arg -> spilled  */
 215                                         var->regoff = parentargs_base + s1 + 1;
 216                                 }
 217                         }
 218
 219                 } else {                                     /* floating args         */
 220                         if (!md->params[p].inmemory) {           /* register arguments    */
 221                                 s2 = rd->argfltregs[s1];
 222                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 223                                         M_FLTMOVE(s2, var->regoff);
 224
 225                                 } else {                                         /* reg arg -> spilled    */
 226                                         M_DST(s2, REG_SP, var->regoff * 8);
 227                                 }
 228
 229                         } else {                                 /* stack arguments       */
 230                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 231                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 232
 233                                 } else {
 234                                         var->regoff = parentargs_base + s1 + 1;
 235                                 }
 236                         }
 237                 }
 238         }  /* end for */
 239
 240         /* save monitorenter argument */
 241
 242 #if defined(USE_THREADS)
 243         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 244                 /* stack offset for monitor argument */
 245
 246                 s1 = rd->memuse;
 247
 248                 if (runverbose) {
 249                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 250
 251                         for (p = 0; p < INT_ARG_CNT; p++)
 252                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 253
 254                         for (p = 0; p < FLT_ARG_CNT; p++)
 255                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 256
 257                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 258                 }
 259
 260                 /* decide which monitor enter function to call */
 261
 262                 if (m->flags & ACC_STATIC) {
 263                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 264                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 265                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 266                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 267                         x86_64_call_reg(cd, REG_ITMP1);
 268
 269                 } else {
 270                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 271                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 272                         codegen_addxnullrefs(cd, cd->mcodeptr);
 273                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 274                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 275                         x86_64_call_reg(cd, REG_ITMP1);
 276                 }
 277
 278                 if (runverbose) {
 279                         for (p = 0; p < INT_ARG_CNT; p++)
 280                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 281
 282                         for (p = 0; p < FLT_ARG_CNT; p++)
 283                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 284
 285                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 286                 }
 287         }
 288 #endif
 289
 290         /* Copy argument registers to stack and call trace function with pointer  */
 291         /* to arguments on stack.                                                 */
 292
 293         if (runverbose || opt_stat) {
 294                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 295
 296                 /* save integer argument registers */
 297
 298                 for (p = 0; p < INT_ARG_CNT; p++)
 299                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 300
 301                 /* save float argument registers */
 302
 303                 for (p = 0; p < FLT_ARG_CNT; p++)
 304                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 305
 306                 /* save temporary registers for leaf methods */
 307
 308                 if (m->isleafmethod) {
 309                         for (p = 0; p < INT_TMP_CNT; p++)
 310                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 311
 312                         for (p = 0; p < FLT_TMP_CNT; p++)
 313                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 314                 }
 315
 316                 if (runverbose) {
 317                         /* show integer hex code for float arguments */
 318
 319                         for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 320                                 /* if the paramtype is a float, we have to right shift all    */
 321                                 /* following integer registers                                */
 322
 323                                 if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 324                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 325                                                 M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 326                                         }
 327
 328                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 329                                         l++;
 330                                 }
 331                         }
 332
 333                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 334                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 335                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 336                         x86_64_call_reg(cd, REG_ITMP1);
 337                 }
 338                 STATS({
 339                         x86_64_mov_imm_reg(cd, (ptrint) compiledinvokation, REG_ITMP1);
 340                         x86_64_call_reg(cd, REG_ITMP1);
 341                 })
 342
 343                 /* restore integer argument registers */
 344
 345                 for (p = 0; p < INT_ARG_CNT; p++)
 346                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 347
 348                 /* restore float argument registers */
 349
 350                 for (p = 0; p < FLT_ARG_CNT; p++)
 351                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 352
 353                 /* restore temporary registers for leaf methods */
 354
 355                 if (m->isleafmethod) {
 356                         for (p = 0; p < INT_TMP_CNT; p++)
 357                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 358
 359                         for (p = 0; p < FLT_TMP_CNT; p++)
 360                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 361                 }
 362
 363                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 364         }
 365
 366         }
 367
 368         /* end of header generation */
 369
 370         /* walk through all basic blocks */
 371         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 372
 373                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 374
 375                 if (bptr->flags >= BBREACHED) {
 376
 377                         /* branch resolving */
 378
 379                         branchref *bref;
 380                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 381                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 382                                                                   bref->branchpos,
 383                                                                   bptr->mpc);
 384                         }
 385
 386                 /* copy interface registers to their destination */
 387
 388                 src = bptr->instack;
 389                 len = bptr->indepth;
 390                 MCODECHECK(512);
 391
 392 #ifdef LSRA
 393                 if (opt_lsra) {
 394                         while (src != NULL) {
 395                                 len--;
 396                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 397                                         if (bptr->type == BBTYPE_SBR) {
 398                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 399                                                 if (!(src->flags & INMEMORY))
 400                                                         d= src->regoff;
 401                                                 else
 402                                                         d=REG_ITMP1;
 403                                                 x86_64_pop_reg(cd, d);
 404                                                 store_reg_to_var_int(src, d);
 405
 406                                         } else if (bptr->type == BBTYPE_EXH) {
 407                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 408                                                 if (!(src->flags & INMEMORY))
 409                                                         d= src->regoff;
 410                                                 else
 411                                                         d=REG_ITMP1;
 412                                                 M_INTMOVE(REG_ITMP1, d);
 413                                                 store_reg_to_var_int(src, d);
 414                                         }
 415                                 }
 416                                 src = src->prev;
 417                         }
 418
 419                 } else {
 420 #endif
 421
 422         while (src != NULL) {
 423                         len--;
 424                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 425                                 if (bptr->type == BBTYPE_SBR) {
 426                                         d = reg_of_var(rd, src, REG_ITMP1);
 427                                         x86_64_pop_reg(cd, d);
 428                                         store_reg_to_var_int(src, d);
 429
 430                                 } else if (bptr->type == BBTYPE_EXH) {
 431                                         d = reg_of_var(rd, src, REG_ITMP1);
 432                                         M_INTMOVE(REG_ITMP1, d);
 433                                         store_reg_to_var_int(src, d);
 434                                 }
 435
 436                         } else {
 437                                 d = reg_of_var(rd, src, REG_ITMP1);
 438                                 if ((src->varkind != STACKVAR)) {
 439                                         s2 = src->type;
 440                                         if (IS_FLT_DBL_TYPE(s2)) {
 441                                                 s1 = rd->interfaces[len][s2].regoff;
 442                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 443                                                         M_FLTMOVE(s1, d);
 444
 445                                                 } else {
 446                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 447                                                 }
 448                                                 store_reg_to_var_flt(src, d);
 449
 450                                         } else {
 451                                                 s1 = rd->interfaces[len][s2].regoff;
 452                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 453                                                         M_INTMOVE(s1, d);
 454
 455                                                 } else {
 456                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 457                                                 }
 458                                                 store_reg_to_var_int(src, d);
 459                                         }
 460                                 }
 461                         }
 462                         src = src->prev;
 463                 }
 464 #ifdef LSRA
 465                 }
 466 #endif
 467                 /* walk through all instructions */
 468
 469                 src = bptr->instack;
 470                 len = bptr->icount;
 471                 currentline=0;
 472
 473                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 474                         if (iptr->line != currentline) {
 475                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 476                                 currentline = iptr->line;
 477                         }
 478
 479                         MCODECHECK(1024);                         /* 1KB should be enough */
 480
 481                         switch (iptr->opc) {
 482                         case ICMD_INLINE_START: /* internal ICMDs                         */
 483                         case ICMD_INLINE_END:
 484                                 break;
 485
 486                         case ICMD_NOP:    /* ...  ==> ...                                 */
 487                                 break;
 488
 489                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 490                                 if (src->flags & INMEMORY) {
 491                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 492
 493                                 } else {
 494                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 495                                 }
 496                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 497                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 498                                 break;
 499
 500                 /* constant operations ************************************************/
 501
 502                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 503                                       /* op1 = 0, val.i = constant                    */
 504
 505                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 506                         if (iptr->val.i == 0) {
 507                                 M_XOR(d, d);
 508                         } else {
 509                                 M_IMOV_IMM(iptr->val.i, d);
 510                         }
 511                         store_reg_to_var_int(iptr->dst, d);
 512                         break;
 513
 514                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 515                                       /* op1 = 0, val.l = constant                    */
 516
 517                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 518                         if (iptr->val.l == 0) {
 519                                 M_XOR(d, d);
 520                         } else {
 521                                 M_MOV_IMM(iptr->val.l, d);
 522                         }
 523                         store_reg_to_var_int(iptr->dst, d);
 524                         break;
 525
 526                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 527                                       /* op1 = 0, val.f = constant                    */
 528
 529                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 530                         disp = dseg_addfloat(cd, iptr->val.f);
 531                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 532                         store_reg_to_var_flt(iptr->dst, d);
 533                         break;
 534
 535                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 536                                       /* op1 = 0, val.d = constant                    */
 537
 538                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 539                         disp = dseg_adddouble(cd, iptr->val.d);
 540                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 541                         store_reg_to_var_flt(iptr->dst, d);
 542                         break;
 543
 544                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 545                                       /* op1 = 0, val.a = constant                    */
 546
 547                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 548
 549                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 550                                 codegen_addpatchref(cd, cd->mcodeptr,
 551                                                                         PATCHER_aconst,
 552                                                                         (unresolved_class *) iptr->target, 0);
 553
 554                                 if (opt_showdisassemble) {
 555                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 556                                 }
 557
 558                                 M_MOV_IMM((ptrint) iptr->val.a, d);
 559
 560                         } else {
 561                                 if (iptr->val.a == 0) {
 562                                         M_XOR(d, d);
 563                                 } else {
 564                                         M_MOV_IMM((ptrint) iptr->val.a, d);
 565                                 }
 566                         }
 567                         store_reg_to_var_int(iptr->dst, d);
 568                         break;
 569
 570
 571                 /* load/store operations **********************************************/
 572
 573                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 574                                       /* op1 = local variable                         */
 575
 576                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 577                         if ((iptr->dst->varkind == LOCALVAR) &&
 578                             (iptr->dst->varnum == iptr->op1)) {
 579                                 break;
 580                         }
 581                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 582                         if (var->flags & INMEMORY) {
 583                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 584                                 store_reg_to_var_int(iptr->dst, d);
 585
 586                         } else {
 587                                 if (iptr->dst->flags & INMEMORY) {
 588                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 589
 590                                 } else {
 591                                         M_INTMOVE(var->regoff, d);
 592                                 }
 593                         }
 594                         break;
 595
 596                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 597                 case ICMD_ALOAD:      /* op1 = local variable                         */
 598
 599                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 600                         if ((iptr->dst->varkind == LOCALVAR) &&
 601                             (iptr->dst->varnum == iptr->op1)) {
 602                                 break;
 603                         }
 604                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 605                         if (var->flags & INMEMORY) {
 606                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 607                                 store_reg_to_var_int(iptr->dst, d);
 608
 609                         } else {
 610                                 if (iptr->dst->flags & INMEMORY) {
 611                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 612
 613                                 } else {
 614                                         M_INTMOVE(var->regoff, d);
 615                                 }
 616                         }
 617                         break;
 618
 619                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 620                 case ICMD_DLOAD:      /* op1 = local variable                         */
 621
 622                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 623                         if ((iptr->dst->varkind == LOCALVAR) &&
 624                             (iptr->dst->varnum == iptr->op1)) {
 625                                 break;
 626                         }
 627                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 628                         if (var->flags & INMEMORY) {
 629                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 630                                 store_reg_to_var_flt(iptr->dst, d);
 631
 632                         } else {
 633                                 if (iptr->dst->flags & INMEMORY) {
 634                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 635
 636                                 } else {
 637                                         M_FLTMOVE(var->regoff, d);
 638                                 }
 639                         }
 640                         break;
 641
 642                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 643                 case ICMD_LSTORE:     /* op1 = local variable                         */
 644                 case ICMD_ASTORE:
 645
 646                         if ((src->varkind == LOCALVAR) &&
 647                             (src->varnum == iptr->op1)) {
 648                                 break;
 649                         }
 650                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 651                         if (var->flags & INMEMORY) {
 652                                 var_to_reg_int(s1, src, REG_ITMP1);
 653                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 654
 655                         } else {
 656                                 var_to_reg_int(s1, src, var->regoff);
 657                                 M_INTMOVE(s1, var->regoff);
 658                         }
 659                         break;
 660
 661                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 662                 case ICMD_DSTORE:     /* op1 = local variable                         */
 663
 664                         if ((src->varkind == LOCALVAR) &&
 665                             (src->varnum == iptr->op1)) {
 666                                 break;
 667                         }
 668                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 669                         if (var->flags & INMEMORY) {
 670                                 var_to_reg_flt(s1, src, REG_FTMP1);
 671                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 672
 673                         } else {
 674                                 var_to_reg_flt(s1, src, var->regoff);
 675                                 M_FLTMOVE(s1, var->regoff);
 676                         }
 677                         break;
 678
 679
 680                 /* pop/dup/swap operations ********************************************/
 681
 682                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 683
 684                 case ICMD_POP:        /* ..., value  ==> ...                          */
 685                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 686                         break;
 687
 688                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 689                         M_COPY(src, iptr->dst);
 690                         break;
 691
 692                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 693
 694                         M_COPY(src,       iptr->dst);
 695                         M_COPY(src->prev, iptr->dst->prev);
 696                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 697                         break;
 698
 699                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 700
 701                         M_COPY(src,             iptr->dst);
 702                         M_COPY(src->prev,       iptr->dst->prev);
 703                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 704                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 705                         break;
 706
 707                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 708
 709                         M_COPY(src,       iptr->dst);
 710                         M_COPY(src->prev, iptr->dst->prev);
 711                         break;
 712
 713                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 714
 715                         M_COPY(src,             iptr->dst);
 716                         M_COPY(src->prev,       iptr->dst->prev);
 717                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 718                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 719                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 720                         break;
 721
 722                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 723
 724                         M_COPY(src,                   iptr->dst);
 725                         M_COPY(src->prev,             iptr->dst->prev);
 726                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 727                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 728                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 729                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 730                         break;
 731
 732                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 733
 734                         M_COPY(src,       iptr->dst->prev);
 735                         M_COPY(src->prev, iptr->dst);
 736                         break;
 737
 738
 739                 /* integer operations *************************************************/
 740
 741                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 742
 743                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 744                         if (iptr->dst->flags & INMEMORY) {
 745                                 if (src->flags & INMEMORY) {
 746                                         if (src->regoff == iptr->dst->regoff) {
 747                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 748
 749                                         } else {
 750                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 751                                                 x86_64_negl_reg(cd, REG_ITMP1);
 752                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 753                                         }
 754
 755                                 } else {
 756                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 757                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 758                                 }
 759
 760                         } else {
 761                                 if (src->flags & INMEMORY) {
 762                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 763                                         x86_64_negl_reg(cd, d);
 764
 765                                 } else {
 766                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 767                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 768                                 }
 769                         }
 770                         break;
 771
 772                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 773
 774                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 775                         if (iptr->dst->flags & INMEMORY) {
 776                                 if (src->flags & INMEMORY) {
 777                                         if (src->regoff == iptr->dst->regoff) {
 778                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 779
 780                                         } else {
 781                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 782                                                 x86_64_neg_reg(cd, REG_ITMP1);
 783                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 784                                         }
 785
 786                                 } else {
 787                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 788                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 789                                 }
 790
 791                         } else {
 792                                 if (src->flags & INMEMORY) {
 793                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 794                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 795
 796                                 } else {
 797                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 798                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 799                                 }
 800                         }
 801                         break;
 802
 803                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 804
 805                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 806                         if (src->flags & INMEMORY) {
 807                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 808
 809                         } else {
 810                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 811                         }
 812                         store_reg_to_var_int(iptr->dst, d);
 813                         break;
 814
 815                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 816
 817                         var_to_reg_int(s1, src, REG_ITMP1);
 818                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 819                         M_INTMOVE(s1, d);
 820                         store_reg_to_var_int(iptr->dst, d);
 821                         break;
 822
 823                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 824
 825                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 826                         if (src->flags & INMEMORY) {
 827                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 828
 829                         } else {
 830                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 831                         }
 832                         store_reg_to_var_int(iptr->dst, d);
 833                         break;
 834
 835                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 836
 837                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 838                         if (src->flags & INMEMORY) {
 839                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 840
 841                         } else {
 842                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 843                         }
 844                         store_reg_to_var_int(iptr->dst, d);
 845                         break;
 846
 847                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 848
 849                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 850                         if (src->flags & INMEMORY) {
 851                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 852
 853                         } else {
 854                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 855                         }
 856                         store_reg_to_var_int(iptr->dst, d);
 857                         break;
 858
 859
 860                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 861
 862                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 863                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 864                         break;
 865
 866                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 867                                       /* val.i = constant                             */
 868
 869                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 870                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 871                         break;
 872
 873                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 874
 875                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 876                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 877                         break;
 878
 879                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 880                                       /* val.l = constant                             */
 881
 882                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 883                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 884                         break;
 885
 886                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 887
 888                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 889                         if (iptr->dst->flags & INMEMORY) {
 890                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 891                                         if (src->prev->regoff == iptr->dst->regoff) {
 892                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 893                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 894
 895                                         } else {
 896                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 897                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 898                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 899                                         }
 900
 901                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 902                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 903                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 904                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 905
 906                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 907                                         if (src->prev->regoff == iptr->dst->regoff) {
 908                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 909
 910                                         } else {
 911                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 912                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 913                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 914                                         }
 915
 916                                 } else {
 917                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 918                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 919                                 }
 920
 921                         } else {
 922                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 923                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 924                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 925
 926                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 927                                         M_INTMOVE(src->prev->regoff, d);
 928                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 929
 930                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 931                                         /* workaround for reg alloc */
 932                                         if (src->regoff == iptr->dst->regoff) {
 933                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 934                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 935                                                 M_INTMOVE(REG_ITMP1, d);
 936
 937                                         } else {
 938                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 939                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 940                                         }
 941
 942                                 } else {
 943                                         /* workaround for reg alloc */
 944                                         if (src->regoff == iptr->dst->regoff) {
 945                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 946                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 947                                                 M_INTMOVE(REG_ITMP1, d);
 948
 949                                         } else {
 950                                                 M_INTMOVE(src->prev->regoff, d);
 951                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 952                                         }
 953                                 }
 954                         }
 955                         break;
 956
 957                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 958                                       /* val.i = constant                             */
 959
 960                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 961                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 962                         break;
 963
 964                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 965
 966                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 967                         if (iptr->dst->flags & INMEMORY) {
 968                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 969                                         if (src->prev->regoff == iptr->dst->regoff) {
 970                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 971                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 972
 973                                         } else {
 974                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 975                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 976                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 977                                         }
 978
 979                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 980                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 981                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 982                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 983
 984                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 985                                         if (src->prev->regoff == iptr->dst->regoff) {
 986                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 987
 988                                         } else {
 989                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 990                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 991                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 992                                         }
 993
 994                                 } else {
 995                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 996                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 997                                 }
 998
 999                         } else {
1000                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1001                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1002                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1003
1004                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1005                                         M_INTMOVE(src->prev->regoff, d);
1006                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1007
1008                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1009                                         /* workaround for reg alloc */
1010                                         if (src->regoff == iptr->dst->regoff) {
1011                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1012                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1013                                                 M_INTMOVE(REG_ITMP1, d);
1014
1015                                         } else {
1016                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1017                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1018                                         }
1019
1020                                 } else {
1021                                         /* workaround for reg alloc */
1022                                         if (src->regoff == iptr->dst->regoff) {
1023                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1024                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1025                                                 M_INTMOVE(REG_ITMP1, d);
1026
1027                                         } else {
1028                                                 M_INTMOVE(src->prev->regoff, d);
1029                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1030                                         }
1031                                 }
1032                         }
1033                         break;
1034
1035                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1036                                       /* val.l = constant                             */
1037
1038                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1039                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1040                         break;
1041
1042                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1043
1044                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1045                         if (iptr->dst->flags & INMEMORY) {
1046                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1047                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1048                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1049                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1050
1051                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1052                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1053                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1054                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1055
1056                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1057                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1058                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1059                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1060
1061                                 } else {
1062                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1063                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1064                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1065                                 }
1066
1067                         } else {
1068                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1069                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1070                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1071
1072                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1073                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1074                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1075
1076                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1077                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1078                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1079
1080                                 } else {
1081                                         if (src->regoff == iptr->dst->regoff) {
1082                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1083
1084                                         } else {
1085                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1086                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1087                                         }
1088                                 }
1089                         }
1090                         break;
1091
1092                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1093                                       /* val.i = constant                             */
1094
1095                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1096                         if (iptr->dst->flags & INMEMORY) {
1097                                 if (src->flags & INMEMORY) {
1098                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1099                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1100
1101                                 } else {
1102                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1103                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1104                                 }
1105
1106                         } else {
1107                                 if (src->flags & INMEMORY) {
1108                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1109
1110                                 } else {
1111                                         if (iptr->val.i == 2) {
1112                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1113                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1114
1115                                         } else {
1116                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1117                                         }
1118                                 }
1119                         }
1120                         break;
1121
1122                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1123
1124                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1125                         if (iptr->dst->flags & INMEMORY) {
1126                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1127                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1128                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1129                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1130
1131                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1132                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1133                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1134                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1135
1136                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1137                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1138                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1139                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1140
1141                                 } else {
1142                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1143                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1144                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1145                                 }
1146
1147                         } else {
1148                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1149                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1150                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1151
1152                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1153                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1154                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1155
1156                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1157                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1158                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1159
1160                                 } else {
1161                                         if (src->regoff == iptr->dst->regoff) {
1162                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1163
1164                                         } else {
1165                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1166                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1167                                         }
1168                                 }
1169                         }
1170                         break;
1171
1172                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1173                                       /* val.l = constant                             */
1174
1175                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1176                         if (iptr->dst->flags & INMEMORY) {
1177                                 if (src->flags & INMEMORY) {
1178                                         if (IS_IMM32(iptr->val.l)) {
1179                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1180
1181                                         } else {
1182                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1183                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1184                                         }
1185                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1186
1187                                 } else {
1188                                         if (IS_IMM32(iptr->val.l)) {
1189                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1190
1191                                         } else {
1192                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1193                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1194                                         }
1195                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1196                                 }
1197
1198                         } else {
1199                                 if (src->flags & INMEMORY) {
1200                                         if (IS_IMM32(iptr->val.l)) {
1201                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1202
1203                                         } else {
1204                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1205                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1206                                         }
1207
1208                                 } else {
1209                                         /* should match in many cases */
1210                                         if (iptr->val.l == 2) {
1211                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1212                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1213
1214                                         } else {
1215                                                 if (IS_IMM32(iptr->val.l)) {
1216                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1217
1218                                                 } else {
1219                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1220                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1221                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1222                                                 }
1223                                         }
1224                                 }
1225                         }
1226                         break;
1227
1228                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1229
1230                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1231                 if (src->prev->flags & INMEMORY) {
1232                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1233
1234                         } else {
1235                                 M_INTMOVE(src->prev->regoff, RAX);
1236                         }
1237
1238                         if (src->flags & INMEMORY) {
1239                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1240
1241                         } else {
1242                                 M_INTMOVE(src->regoff, REG_ITMP3);
1243                         }
1244                         gen_div_check(src);
1245
1246                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1247                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1248                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1249                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1250
1251                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1252                         x86_64_cltd(cd);
1253                         x86_64_idivl_reg(cd, REG_ITMP3);
1254
1255                         if (iptr->dst->flags & INMEMORY) {
1256                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1257                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1258
1259                         } else {
1260                                 M_INTMOVE(RAX, iptr->dst->regoff);
1261
1262                                 if (iptr->dst->regoff != RDX) {
1263                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1264                                 }
1265                         }
1266                         break;
1267
1268                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1269                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1270                         if (src->prev->flags & INMEMORY) {
1271                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1272
1273                         } else {
1274                                 M_INTMOVE(src->prev->regoff, RAX);
1275                         }
1276
1277                         if (src->flags & INMEMORY) {
1278                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1279
1280                         } else {
1281                                 M_INTMOVE(src->regoff, REG_ITMP3);
1282                         }
1283                         gen_div_check(src);
1284
1285                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1286
1287                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1288                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1289
1290
1291                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1292                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1293                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1294
1295                         x86_64_cltd(cd);
1296                         x86_64_idivl_reg(cd, REG_ITMP3);
1297
1298                         if (iptr->dst->flags & INMEMORY) {
1299                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1300                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1301
1302                         } else {
1303                                 M_INTMOVE(RDX, iptr->dst->regoff);
1304
1305                                 if (iptr->dst->regoff != RDX) {
1306                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1307                                 }
1308                         }
1309                         break;
1310
1311                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1312                                       /* val.i = constant                             */
1313
1314                         var_to_reg_int(s1, src, REG_ITMP1);
1315                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1316                         M_INTMOVE(s1, REG_ITMP1);
1317                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1318                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1319                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1320                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1321                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1322                         store_reg_to_var_int(iptr->dst, d);
1323                         break;
1324
1325                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1326                                       /* val.i = constant                             */
1327
1328                         var_to_reg_int(s1, src, REG_ITMP1);
1329                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1330                         M_INTMOVE(s1, REG_ITMP1);
1331                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1332                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1333                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1334                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1335                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1336                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1337                         store_reg_to_var_int(iptr->dst, d);
1338                         break;
1339
1340
1341                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1342
1343                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1344
1345                 if (src->prev->flags & INMEMORY) {
1346                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1347
1348                         } else {
1349                                 M_INTMOVE(src->prev->regoff, RAX);
1350                         }
1351
1352                         if (src->flags & INMEMORY) {
1353                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1354
1355                         } else {
1356                                 M_INTMOVE(src->regoff, REG_ITMP3);
1357                         }
1358                         gen_div_check(src);
1359
1360                         /* check as described in jvm spec */
1361                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1362                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1363                         M_BNE(4 + 6);
1364                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1365                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1366
1367                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1368                         x86_64_cqto(cd);
1369                         x86_64_idiv_reg(cd, REG_ITMP3);
1370
1371                         if (iptr->dst->flags & INMEMORY) {
1372                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1373                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1374
1375                         } else {
1376                                 M_INTMOVE(RAX, iptr->dst->regoff);
1377
1378                                 if (iptr->dst->regoff != RDX) {
1379                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1380                                 }
1381                         }
1382                         break;
1383
1384                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1385
1386                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1387                         if (src->prev->flags & INMEMORY) {
1388                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1389
1390                         } else {
1391                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1392                         }
1393
1394                         if (src->flags & INMEMORY) {
1395                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1396
1397                         } else {
1398                                 M_INTMOVE(src->regoff, REG_ITMP3);
1399                         }
1400                         gen_div_check(src);
1401
1402                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1403
1404                         /* check as described in jvm spec */
1405                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1406                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1407                         M_BNE(3 + 4 + 6);
1408
1409 #if 0
1410                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1411 #endif
1412                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1413                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1414                         M_BEQ(2 + 3);                                          /* 6 bytes */
1415
1416                         x86_64_cqto(cd);
1417                         x86_64_idiv_reg(cd, REG_ITMP3);
1418
1419                         if (iptr->dst->flags & INMEMORY) {
1420                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1421                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1422
1423                         } else {
1424                                 M_INTMOVE(RDX, iptr->dst->regoff);
1425
1426                                 if (iptr->dst->regoff != RDX) {
1427                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1428                                 }
1429                         }
1430                         break;
1431
1432                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1433                                       /* val.i = constant                             */
1434
1435                         var_to_reg_int(s1, src, REG_ITMP1);
1436                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1437                         M_INTMOVE(s1, REG_ITMP1);
1438                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1439                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1440                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1441                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1442                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1443                         store_reg_to_var_int(iptr->dst, d);
1444                         break;
1445
1446                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1447                                       /* val.l = constant                             */
1448
1449                         var_to_reg_int(s1, src, REG_ITMP1);
1450                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1451                         M_INTMOVE(s1, REG_ITMP1);
1452                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1453                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1454                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1455                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1456                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1457                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1458                         store_reg_to_var_int(iptr->dst, d);
1459                         break;
1460
1461                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1462
1463                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1464                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1465                         break;
1466
1467                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1468                                       /* val.i = constant                             */
1469
1470                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1471                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1472                         break;
1473
1474                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1475
1476                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1477                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1478                         break;
1479
1480                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1481                                       /* val.i = constant                             */
1482
1483                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1484                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1485                         break;
1486
1487                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1488
1489                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1490                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1491                         break;
1492
1493                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1494                                       /* val.i = constant                             */
1495
1496                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1497                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1498                         break;
1499
1500                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1501
1502                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1503                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1504                         break;
1505
1506         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1507                                           /* val.i = constant                             */
1508
1509                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1510                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1511                         break;
1512
1513                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1514
1515                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1516                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1517                         break;
1518
1519                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1520                                       /* val.i = constant                             */
1521
1522                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1523                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1524                         break;
1525
1526                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1527
1528                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1529                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1530                         break;
1531
1532                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1533                                       /* val.l = constant                             */
1534
1535                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1536                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1537                         break;
1538
1539                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1540
1541                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1542                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1543                         break;
1544
1545                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1546                                       /* val.i = constant                             */
1547
1548                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1549                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1550                         break;
1551
1552                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1553
1554                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1555                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1556                         break;
1557
1558                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1559                                       /* val.l = constant                             */
1560
1561                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1562                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1563                         break;
1564
1565                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1566
1567                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1568                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1569                         break;
1570
1571                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1572                                       /* val.i = constant                             */
1573
1574                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1575                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1576                         break;
1577
1578                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1579
1580                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1581                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1582                         break;
1583
1584                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1585                                       /* val.l = constant                             */
1586
1587                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1588                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1589                         break;
1590
1591                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1592
1593                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1594                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1595                         break;
1596
1597                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1598                                       /* val.i = constant                             */
1599
1600                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1601                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1602                         break;
1603
1604                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1605
1606                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1607                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1608                         break;
1609
1610                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1611                                       /* val.l = constant                             */
1612
1613                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1614                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1615                         break;
1616
1617
1618                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1619                                       /* op1 = variable, val.i = constant             */
1620
1621                         /* using inc and dec is definitely faster than add -- tested      */
1622                         /* with sieve                                                     */
1623
1624                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1625                         d = var->regoff;
1626                         if (var->flags & INMEMORY) {
1627                                 if (iptr->val.i == 1) {
1628                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1629
1630                                 } else if (iptr->val.i == -1) {
1631                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1632
1633                                 } else {
1634                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1635                                 }
1636
1637                         } else {
1638                                 if (iptr->val.i == 1) {
1639                                         x86_64_incl_reg(cd, d);
1640
1641                                 } else if (iptr->val.i == -1) {
1642                                         x86_64_decl_reg(cd, d);
1643
1644                                 } else {
1645                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1646                                 }
1647                         }
1648                         break;
1649
1650
1651                 /* floating operations ************************************************/
1652
1653                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1654
1655                         var_to_reg_flt(s1, src, REG_FTMP1);
1656                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1657                         disp = dseg_adds4(cd, 0x80000000);
1658                         M_FLTMOVE(s1, d);
1659                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1660                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1661                         store_reg_to_var_flt(iptr->dst, d);
1662                         break;
1663
1664                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1665
1666                         var_to_reg_flt(s1, src, REG_FTMP1);
1667                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1668                         disp = dseg_adds8(cd, 0x8000000000000000);
1669                         M_FLTMOVE(s1, d);
1670                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1671                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1672                         store_reg_to_var_flt(iptr->dst, d);
1673                         break;
1674
1675                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1676
1677                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1678                         var_to_reg_flt(s2, src, REG_FTMP2);
1679                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1680                         if (s1 == d) {
1681                                 x86_64_addss_reg_reg(cd, s2, d);
1682                         } else if (s2 == d) {
1683                                 x86_64_addss_reg_reg(cd, s1, d);
1684                         } else {
1685                                 M_FLTMOVE(s1, d);
1686                                 x86_64_addss_reg_reg(cd, s2, d);
1687                         }
1688                         store_reg_to_var_flt(iptr->dst, d);
1689                         break;
1690
1691                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1692
1693                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1694                         var_to_reg_flt(s2, src, REG_FTMP2);
1695                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1696                         if (s1 == d) {
1697                                 x86_64_addsd_reg_reg(cd, s2, d);
1698                         } else if (s2 == d) {
1699                                 x86_64_addsd_reg_reg(cd, s1, d);
1700                         } else {
1701                                 M_FLTMOVE(s1, d);
1702                                 x86_64_addsd_reg_reg(cd, s2, d);
1703                         }
1704                         store_reg_to_var_flt(iptr->dst, d);
1705                         break;
1706
1707                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1708
1709                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1710                         var_to_reg_flt(s2, src, REG_FTMP2);
1711                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1712                         if (s2 == d) {
1713                                 M_FLTMOVE(s2, REG_FTMP2);
1714                                 s2 = REG_FTMP2;
1715                         }
1716                         M_FLTMOVE(s1, d);
1717                         x86_64_subss_reg_reg(cd, s2, d);
1718                         store_reg_to_var_flt(iptr->dst, d);
1719                         break;
1720
1721                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1722
1723                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1724                         var_to_reg_flt(s2, src, REG_FTMP2);
1725                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1726                         if (s2 == d) {
1727                                 M_FLTMOVE(s2, REG_FTMP2);
1728                                 s2 = REG_FTMP2;
1729                         }
1730                         M_FLTMOVE(s1, d);
1731                         x86_64_subsd_reg_reg(cd, s2, d);
1732                         store_reg_to_var_flt(iptr->dst, d);
1733                         break;
1734
1735                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1736
1737                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1738                         var_to_reg_flt(s2, src, REG_FTMP2);
1739                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1740                         if (s1 == d) {
1741                                 x86_64_mulss_reg_reg(cd, s2, d);
1742                         } else if (s2 == d) {
1743                                 x86_64_mulss_reg_reg(cd, s1, d);
1744                         } else {
1745                                 M_FLTMOVE(s1, d);
1746                                 x86_64_mulss_reg_reg(cd, s2, d);
1747                         }
1748                         store_reg_to_var_flt(iptr->dst, d);
1749                         break;
1750
1751                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1752
1753                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1754                         var_to_reg_flt(s2, src, REG_FTMP2);
1755                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1756                         if (s1 == d) {
1757                                 x86_64_mulsd_reg_reg(cd, s2, d);
1758                         } else if (s2 == d) {
1759                                 x86_64_mulsd_reg_reg(cd, s1, d);
1760                         } else {
1761                                 M_FLTMOVE(s1, d);
1762                                 x86_64_mulsd_reg_reg(cd, s2, d);
1763                         }
1764                         store_reg_to_var_flt(iptr->dst, d);
1765                         break;
1766
1767                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1768
1769                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1770                         var_to_reg_flt(s2, src, REG_FTMP2);
1771                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1772                         if (s2 == d) {
1773                                 M_FLTMOVE(s2, REG_FTMP2);
1774                                 s2 = REG_FTMP2;
1775                         }
1776                         M_FLTMOVE(s1, d);
1777                         x86_64_divss_reg_reg(cd, s2, d);
1778                         store_reg_to_var_flt(iptr->dst, d);
1779                         break;
1780
1781                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1782
1783                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1784                         var_to_reg_flt(s2, src, REG_FTMP2);
1785                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1786                         if (s2 == d) {
1787                                 M_FLTMOVE(s2, REG_FTMP2);
1788                                 s2 = REG_FTMP2;
1789                         }
1790                         M_FLTMOVE(s1, d);
1791                         x86_64_divsd_reg_reg(cd, s2, d);
1792                         store_reg_to_var_flt(iptr->dst, d);
1793                         break;
1794
1795                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1796
1797                         var_to_reg_int(s1, src, REG_ITMP1);
1798                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1799                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1800                         store_reg_to_var_flt(iptr->dst, d);
1801                         break;
1802
1803                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1804
1805                         var_to_reg_int(s1, src, REG_ITMP1);
1806                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1807                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1808                         store_reg_to_var_flt(iptr->dst, d);
1809                         break;
1810
1811                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1812
1813                         var_to_reg_int(s1, src, REG_ITMP1);
1814                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1815                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1816                         store_reg_to_var_flt(iptr->dst, d);
1817                         break;
1818
1819                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1820
1821                         var_to_reg_int(s1, src, REG_ITMP1);
1822                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1823                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1824                         store_reg_to_var_flt(iptr->dst, d);
1825                         break;
1826
1827                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1828
1829                         var_to_reg_flt(s1, src, REG_FTMP1);
1830                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1831                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1832                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1833                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1834                         x86_64_jcc(cd, X86_64_CC_NE, a);
1835                         M_FLTMOVE(s1, REG_FTMP1);
1836                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2i, REG_ITMP2);
1837                         x86_64_call_reg(cd, REG_ITMP2);
1838                         M_INTMOVE(REG_RESULT, d);
1839                         store_reg_to_var_int(iptr->dst, d);
1840                         break;
1841
1842                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1843
1844                         var_to_reg_flt(s1, src, REG_FTMP1);
1845                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1846                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1847                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1848                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1849                         x86_64_jcc(cd, X86_64_CC_NE, a);
1850                         M_FLTMOVE(s1, REG_FTMP1);
1851                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2i, REG_ITMP2);
1852                         x86_64_call_reg(cd, REG_ITMP2);
1853                         M_INTMOVE(REG_RESULT, d);
1854                         store_reg_to_var_int(iptr->dst, d);
1855                         break;
1856
1857                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1858
1859                         var_to_reg_flt(s1, src, REG_FTMP1);
1860                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1861                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1862                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1863                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1864                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1865                         x86_64_jcc(cd, X86_64_CC_NE, a);
1866                         M_FLTMOVE(s1, REG_FTMP1);
1867                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_f2l, REG_ITMP2);
1868                         x86_64_call_reg(cd, REG_ITMP2);
1869                         M_INTMOVE(REG_RESULT, d);
1870                         store_reg_to_var_int(iptr->dst, d);
1871                         break;
1872
1873                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1874
1875                         var_to_reg_flt(s1, src, REG_FTMP1);
1876                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1877                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1878                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1879                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1880                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1881                         x86_64_jcc(cd, X86_64_CC_NE, a);
1882                         M_FLTMOVE(s1, REG_FTMP1);
1883                         x86_64_mov_imm_reg(cd, (s8) asm_builtin_d2l, REG_ITMP2);
1884                         x86_64_call_reg(cd, REG_ITMP2);
1885                         M_INTMOVE(REG_RESULT, d);
1886                         store_reg_to_var_int(iptr->dst, d);
1887                         break;
1888
1889                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1890
1891                         var_to_reg_flt(s1, src, REG_FTMP1);
1892                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1893                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1894                         store_reg_to_var_flt(iptr->dst, d);
1895                         break;
1896
1897                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1898
1899                         var_to_reg_flt(s1, src, REG_FTMP1);
1900                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1901                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1902                         store_reg_to_var_flt(iptr->dst, d);
1903                         break;
1904
1905                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1906                                           /* == => 0, < => 1, > => -1 */
1907
1908                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1909                         var_to_reg_flt(s2, src, REG_FTMP2);
1910                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1911                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1912                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1913                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1914                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1915                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1916                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1917                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1918                         store_reg_to_var_int(iptr->dst, d);
1919                         break;
1920
1921                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1922                                           /* == => 0, < => 1, > => -1 */
1923
1924                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1925                         var_to_reg_flt(s2, src, REG_FTMP2);
1926                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1927                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1928                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1929                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1930                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1931                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1932                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1933                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1934                         store_reg_to_var_int(iptr->dst, d);
1935                         break;
1936
1937                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1938                                           /* == => 0, < => 1, > => -1 */
1939
1940                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1941                         var_to_reg_flt(s2, src, REG_FTMP2);
1942                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1943                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1944                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1945                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1946                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1947                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1948                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1949                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1950                         store_reg_to_var_int(iptr->dst, d);
1951                         break;
1952
1953                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1954                                           /* == => 0, < => 1, > => -1 */
1955
1956                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1957                         var_to_reg_flt(s2, src, REG_FTMP2);
1958                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1959                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1960                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1961                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1962                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1963                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1964                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1965                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1966                         store_reg_to_var_int(iptr->dst, d);
1967                         break;
1968
1969
1970                 /* memory operations **************************************************/
1971
1972                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1973
1974                         var_to_reg_int(s1, src, REG_ITMP1);
1975                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1976                         gen_nullptr_check(s1);
1977                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1978                         store_reg_to_var_int(iptr->dst, d);
1979                         break;
1980
1981                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1982
1983                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1984                         var_to_reg_int(s2, src, REG_ITMP2);
1985                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1986                         if (iptr->op1 == 0) {
1987                                 gen_nullptr_check(s1);
1988                                 gen_bound_check;
1989                         }
1990                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1991                         store_reg_to_var_int(iptr->dst, d);
1992                         break;
1993
1994                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1995
1996                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1997                         var_to_reg_int(s2, src, REG_ITMP2);
1998                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1999                         if (iptr->op1 == 0) {
2000                                 gen_nullptr_check(s1);
2001                                 gen_bound_check;
2002                         }
2003                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2004                         store_reg_to_var_int(iptr->dst, d);
2005                         break;
2006
2007                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2008
2009                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2010                         var_to_reg_int(s2, src, REG_ITMP2);
2011                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2012                         if (iptr->op1 == 0) {
2013                                 gen_nullptr_check(s1);
2014                                 gen_bound_check;
2015                         }
2016                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2017                         store_reg_to_var_int(iptr->dst, d);
2018                         break;
2019
2020                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2021
2022                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2023                         var_to_reg_int(s2, src, REG_ITMP2);
2024                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2025                         if (iptr->op1 == 0) {
2026                                 gen_nullptr_check(s1);
2027                                 gen_bound_check;
2028                         }
2029                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2030                         store_reg_to_var_flt(iptr->dst, d);
2031                         break;
2032
2033                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2034
2035                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2036                         var_to_reg_int(s2, src, REG_ITMP2);
2037                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2038                         if (iptr->op1 == 0) {
2039                                 gen_nullptr_check(s1);
2040                                 gen_bound_check;
2041                         }
2042                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2043                         store_reg_to_var_flt(iptr->dst, d);
2044                         break;
2045
2046                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2047
2048                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2049                         var_to_reg_int(s2, src, REG_ITMP2);
2050                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2051                         if (iptr->op1 == 0) {
2052                                 gen_nullptr_check(s1);
2053                                 gen_bound_check;
2054                         }
2055                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2056                         store_reg_to_var_int(iptr->dst, d);
2057                         break;
2058
2059                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2060
2061                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2062                         var_to_reg_int(s2, src, REG_ITMP2);
2063                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2064                         if (iptr->op1 == 0) {
2065                                 gen_nullptr_check(s1);
2066                                 gen_bound_check;
2067                         }
2068                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2069                         store_reg_to_var_int(iptr->dst, d);
2070                         break;
2071
2072                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2073
2074                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2075                         var_to_reg_int(s2, src, REG_ITMP2);
2076                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2077                         if (iptr->op1 == 0) {
2078                                 gen_nullptr_check(s1);
2079                                 gen_bound_check;
2080                         }
2081                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2082                         store_reg_to_var_int(iptr->dst, d);
2083                         break;
2084
2085
2086                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2087
2088                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2089                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2090                         if (iptr->op1 == 0) {
2091                                 gen_nullptr_check(s1);
2092                                 gen_bound_check;
2093                         }
2094                         var_to_reg_int(s3, src, REG_ITMP3);
2095                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2096                         break;
2097
2098                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2099
2100                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2101                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2102                         if (iptr->op1 == 0) {
2103                                 gen_nullptr_check(s1);
2104                                 gen_bound_check;
2105                         }
2106                         var_to_reg_int(s3, src, REG_ITMP3);
2107                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2108                         break;
2109
2110                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2111
2112                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2113                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2114                         if (iptr->op1 == 0) {
2115                                 gen_nullptr_check(s1);
2116                                 gen_bound_check;
2117                         }
2118                         var_to_reg_flt(s3, src, REG_FTMP3);
2119                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2120                         break;
2121
2122                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2123
2124                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2125                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2126                         if (iptr->op1 == 0) {
2127                                 gen_nullptr_check(s1);
2128                                 gen_bound_check;
2129                         }
2130                         var_to_reg_flt(s3, src, REG_FTMP3);
2131                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2132                         break;
2133
2134                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2135
2136                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2137                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2138                         if (iptr->op1 == 0) {
2139                                 gen_nullptr_check(s1);
2140                                 gen_bound_check;
2141                         }
2142                         var_to_reg_int(s3, src, REG_ITMP3);
2143                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2144                         break;
2145
2146                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2147
2148                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2149                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2150                         if (iptr->op1 == 0) {
2151                                 gen_nullptr_check(s1);
2152                                 gen_bound_check;
2153                         }
2154                         var_to_reg_int(s3, src, REG_ITMP3);
2155                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2156                         break;
2157
2158                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2159
2160                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2161                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2162                         if (iptr->op1 == 0) {
2163                                 gen_nullptr_check(s1);
2164                                 gen_bound_check;
2165                         }
2166                         var_to_reg_int(s3, src, REG_ITMP3);
2167                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2168                         break;
2169
2170                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2171
2172                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2173                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2174                         if (iptr->op1 == 0) {
2175                                 gen_nullptr_check(s1);
2176                                 gen_bound_check;
2177                         }
2178                         var_to_reg_int(s3, src, REG_ITMP3);
2179
2180                         M_MOV(s1, rd->argintregs[0]);
2181                         M_MOV(s3, rd->argintregs[1]);
2182                         M_MOV_IMM((ptrint) BUILTIN_canstore, REG_ITMP1);
2183                         M_CALL(REG_ITMP1);
2184                         M_TEST(REG_RESULT);
2185                         M_BEQ(0);
2186                         codegen_addxstorerefs(cd, cd->mcodeptr);
2187
2188                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2189                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2190                         var_to_reg_int(s3, src, REG_ITMP3);
2191                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2192                         break;
2193
2194
2195                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2196
2197                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2198                         var_to_reg_int(s2, src, REG_ITMP2);
2199                         if (iptr->op1 == 0) {
2200                                 gen_nullptr_check(s1);
2201                                 gen_bound_check;
2202                         }
2203                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2204                         break;
2205
2206                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2207
2208                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2209                         var_to_reg_int(s2, src, REG_ITMP2);
2210                         if (iptr->op1 == 0) {
2211                                 gen_nullptr_check(s1);
2212                                 gen_bound_check;
2213                         }
2214
2215                         if (IS_IMM32(iptr->val.l)) {
2216                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2217
2218                         } else {
2219                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2220                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2221                         }
2222                         break;
2223
2224                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2225
2226                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2227                         var_to_reg_int(s2, src, REG_ITMP2);
2228                         if (iptr->op1 == 0) {
2229                                 gen_nullptr_check(s1);
2230                                 gen_bound_check;
2231                         }
2232                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2233                         break;
2234
2235                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2236
2237                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2238                         var_to_reg_int(s2, src, REG_ITMP2);
2239                         if (iptr->op1 == 0) {
2240                                 gen_nullptr_check(s1);
2241                                 gen_bound_check;
2242                         }
2243                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2244                         break;
2245
2246                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2247
2248                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2249                         var_to_reg_int(s2, src, REG_ITMP2);
2250                         if (iptr->op1 == 0) {
2251                                 gen_nullptr_check(s1);
2252                                 gen_bound_check;
2253                         }
2254                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2255                         break;
2256
2257                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2258
2259                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2260                         var_to_reg_int(s2, src, REG_ITMP2);
2261                         if (iptr->op1 == 0) {
2262                                 gen_nullptr_check(s1);
2263                                 gen_bound_check;
2264                         }
2265                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2266                         break;
2267
2268
2269                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2270                                       /* op1 = type, val.a = field address            */
2271
2272                         if (!iptr->val.a) {
2273                                 disp = dseg_addaddress(cd, NULL);
2274
2275                                 codegen_addpatchref(cd, cd->mcodeptr,
2276                                                                         PATCHER_get_putstatic,
2277                                                                         (unresolved_field *) iptr->target, disp);
2278
2279                                 if (opt_showdisassemble) {
2280                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2281                                 }
2282
2283                         } else {
2284                                 fieldinfo *fi = iptr->val.a;
2285
2286                                 disp = dseg_addaddress(cd, &(fi->value));
2287
2288                                 if (!fi->class->initialized) {
2289                                         codegen_addpatchref(cd, cd->mcodeptr,
2290                                                                                 PATCHER_clinit, fi->class, 0);
2291
2292                                         if (opt_showdisassemble) {
2293                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2294                                         }
2295                                 }
2296                         }
2297
2298                         /* This approach is much faster than moving the field address     */
2299                         /* inline into a register. */
2300                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2301                         switch (iptr->op1) {
2302                         case TYPE_INT:
2303                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2304                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2305                                 store_reg_to_var_int(iptr->dst, d);
2306                                 break;
2307                         case TYPE_LNG:
2308                         case TYPE_ADR:
2309                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2310                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2311                                 store_reg_to_var_int(iptr->dst, d);
2312                                 break;
2313                         case TYPE_FLT:
2314                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2315                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2316                                 store_reg_to_var_flt(iptr->dst, d);
2317                                 break;
2318                         case TYPE_DBL:
2319                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2320                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2321                                 store_reg_to_var_flt(iptr->dst, d);
2322                                 break;
2323                         }
2324                         break;
2325
2326                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2327                                       /* op1 = type, val.a = field address            */
2328
2329                         if (!iptr->val.a) {
2330                                 disp = dseg_addaddress(cd, NULL);
2331
2332                                 codegen_addpatchref(cd, cd->mcodeptr,
2333                                                                         PATCHER_get_putstatic,
2334                                                                         (unresolved_field *) iptr->target, disp);
2335
2336                                 if (opt_showdisassemble) {
2337                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2338                                 }
2339
2340                         } else {
2341                                 fieldinfo *fi = iptr->val.a;
2342
2343                                 disp = dseg_addaddress(cd, &(fi->value));
2344
2345                                 if (!fi->class->initialized) {
2346                                         codegen_addpatchref(cd, cd->mcodeptr,
2347                                                                                 PATCHER_clinit, fi->class, 0);
2348
2349                                         if (opt_showdisassemble) {
2350                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2351                                         }
2352                                 }
2353                         }
2354
2355                         /* This approach is much faster than moving the field address     */
2356                         /* inline into a register. */
2357                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2358                         switch (iptr->op1) {
2359                         case TYPE_INT:
2360                                 var_to_reg_int(s2, src, REG_ITMP1);
2361                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2362                                 break;
2363                         case TYPE_LNG:
2364                         case TYPE_ADR:
2365                                 var_to_reg_int(s2, src, REG_ITMP1);
2366                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2367                                 break;
2368                         case TYPE_FLT:
2369                                 var_to_reg_flt(s2, src, REG_FTMP1);
2370                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2371                                 break;
2372                         case TYPE_DBL:
2373                                 var_to_reg_flt(s2, src, REG_FTMP1);
2374                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2375                                 break;
2376                         }
2377                         break;
2378
2379                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2380                                           /* val = value (in current instruction)     */
2381                                           /* op1 = type, val.a = field address (in    */
2382                                           /* following NOP)                           */
2383
2384                         if (!iptr[1].val.a) {
2385                                 disp = dseg_addaddress(cd, NULL);
2386
2387                                 codegen_addpatchref(cd, cd->mcodeptr,
2388                                                                         PATCHER_get_putstatic,
2389                                                                         (unresolved_field *) iptr[1].target, disp);
2390
2391                                 if (opt_showdisassemble) {
2392                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2393                                 }
2394
2395                         } else {
2396                                 fieldinfo *fi = iptr[1].val.a;
2397
2398                                 disp = dseg_addaddress(cd, &(fi->value));
2399
2400                                 if (!fi->class->initialized) {
2401                                         codegen_addpatchref(cd, cd->mcodeptr,
2402                                                                                 PATCHER_clinit, fi->class, 0);
2403
2404                                         if (opt_showdisassemble) {
2405                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2406                                         }
2407                                 }
2408                         }
2409
2410                         /* This approach is much faster than moving the field address     */
2411                         /* inline into a register. */
2412                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
2413                         switch (iptr->op1) {
2414                         case TYPE_INT:
2415                         case TYPE_FLT:
2416                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2417                                 break;
2418                         case TYPE_LNG:
2419                         case TYPE_ADR:
2420                         case TYPE_DBL:
2421                                 if (IS_IMM32(iptr->val.l)) {
2422                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2423                                 } else {
2424                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2425                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2426                                 }
2427                                 break;
2428                         }
2429                         break;
2430
2431                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2432                                       /* op1 = type, val.i = field offset             */
2433
2434                         var_to_reg_int(s1, src, REG_ITMP1);
2435                         gen_nullptr_check(s1);
2436
2437                         if (!iptr->val.a) {
2438                                 codegen_addpatchref(cd, cd->mcodeptr,
2439                                                                         PATCHER_get_putfield,
2440                                                                         (unresolved_field *) iptr->target, 0);
2441
2442                                 if (opt_showdisassemble) {
2443                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2444                                 }
2445
2446                                 a = 0;
2447
2448                         } else {
2449                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2450                         }
2451
2452                         switch (iptr->op1) {
2453                         case TYPE_INT:
2454                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2455                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2456                                 store_reg_to_var_int(iptr->dst, d);
2457                                 break;
2458                         case TYPE_LNG:
2459                         case TYPE_ADR:
2460                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2461                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2462                                 store_reg_to_var_int(iptr->dst, d);
2463                                 break;
2464                         case TYPE_FLT:
2465                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2466                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2467                                 store_reg_to_var_flt(iptr->dst, d);
2468                                 break;
2469                         case TYPE_DBL:
2470                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2471                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2472                                 store_reg_to_var_flt(iptr->dst, d);
2473                                 break;
2474                         }
2475                         break;
2476
2477                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2478                                       /* op1 = type, val.i = field offset             */
2479
2480                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2481                         gen_nullptr_check(s1);
2482                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2483                                 var_to_reg_int(s2, src, REG_ITMP2);
2484                         } else {
2485                                 var_to_reg_flt(s2, src, REG_FTMP2);
2486                         }
2487
2488                         if (!iptr->val.a) {
2489                                 codegen_addpatchref(cd, cd->mcodeptr,
2490                                                                         PATCHER_get_putfield,
2491                                                                         (unresolved_field *) iptr->target, 0);
2492
2493                                 if (opt_showdisassemble) {
2494                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2495                                 }
2496
2497                                 a = 0;
2498
2499                         } else {
2500                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2501                         }
2502
2503                         switch (iptr->op1) {
2504                         case TYPE_INT:
2505                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2506                                 break;
2507                         case TYPE_LNG:
2508                         case TYPE_ADR:
2509                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2510                                 break;
2511                         case TYPE_FLT:
2512                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2513                                 break;
2514                         case TYPE_DBL:
2515                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2516                                 break;
2517                         }
2518                         break;
2519
2520                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2521                                           /* val = value (in current instruction)     */
2522                                           /* op1 = type, val.a = field address (in    */
2523                                           /* following NOP)                           */
2524
2525                         var_to_reg_int(s1, src, REG_ITMP1);
2526                         gen_nullptr_check(s1);
2527
2528                         if (!iptr[1].val.a) {
2529                                 codegen_addpatchref(cd, cd->mcodeptr,
2530                                                                         PATCHER_putfieldconst,
2531                                                                         (unresolved_field *) iptr[1].target, 0);
2532
2533                                 if (opt_showdisassemble) {
2534                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2535                                 }
2536
2537                                 a = 0;
2538
2539                         } else {
2540                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2541                         }
2542
2543                         switch (iptr->op1) {
2544                         case TYPE_INT:
2545                         case TYPE_FLT:
2546                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2547                                 break;
2548                         case TYPE_LNG:
2549                         case TYPE_ADR:
2550                         case TYPE_DBL:
2551                                 /* We can only optimize the move, if the class is resolved.   */
2552                                 /* Otherwise we don't know what to patch.                     */
2553                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2554                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2555                                 } else {
2556                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2557                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2558                                 }
2559                                 break;
2560                         }
2561                         break;
2562
2563
2564                 /* branch operations **************************************************/
2565
2566                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2567
2568                         var_to_reg_int(s1, src, REG_ITMP1);
2569                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2570
2571                         if (iptr->val.a) {
2572                                 codegen_addpatchref(cd, cd->mcodeptr,
2573                                                                         PATCHER_athrow_areturn,
2574                                                                         (unresolved_class *) iptr->val.a, 0);
2575
2576                                 if (opt_showdisassemble) {
2577                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2578                                 }
2579                         }
2580
2581                         M_CALL_IMM(0);                            /* passing exception pc */
2582                         M_POP(REG_ITMP2_XPC);
2583
2584                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
2585                         M_JMP(REG_ITMP3);
2586                         break;
2587
2588                 case ICMD_GOTO:         /* ... ==> ...                                */
2589                                         /* op1 = target JavaVM pc                     */
2590
2591                         M_JMP_IMM(0);
2592                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2593                         break;
2594
2595                 case ICMD_JSR:          /* ... ==> ...                                */
2596                                         /* op1 = target JavaVM pc                     */
2597
2598                         M_CALL_IMM(0);
2599                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2600                         break;
2601
2602                 case ICMD_RET:          /* ... ==> ...                                */
2603                                         /* op1 = local variable                       */
2604
2605                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2606                         var_to_reg_int(s1, var, REG_ITMP1);
2607                         M_JMP(s1);
2608                         break;
2609
2610                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2611                                         /* op1 = target JavaVM pc                     */
2612
2613                         if (src->flags & INMEMORY) {
2614                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2615
2616                         } else {
2617                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2618                         }
2619                         x86_64_jcc(cd, X86_64_CC_E, 0);
2620                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2621                         break;
2622
2623                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2624                                         /* op1 = target JavaVM pc                     */
2625
2626                         if (src->flags & INMEMORY) {
2627                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2628
2629                         } else {
2630                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2631                         }
2632                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2633                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2634                         break;
2635
2636                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2637                                         /* op1 = target JavaVM pc, val.i = constant   */
2638
2639                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2640                         break;
2641
2642                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2643                                         /* op1 = target JavaVM pc, val.i = constant   */
2644
2645                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2646                         break;
2647
2648                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2649                                         /* op1 = target JavaVM pc, val.i = constant   */
2650
2651                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2652                         break;
2653
2654                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2655                                         /* op1 = target JavaVM pc, val.i = constant   */
2656
2657                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2658                         break;
2659
2660                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2661                                         /* op1 = target JavaVM pc, val.i = constant   */
2662
2663                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2664                         break;
2665
2666                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2667                                         /* op1 = target JavaVM pc, val.i = constant   */
2668
2669                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2670                         break;
2671
2672                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2673                                         /* op1 = target JavaVM pc, val.l = constant   */
2674
2675                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2676                         break;
2677
2678                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2679                                         /* op1 = target JavaVM pc, val.l = constant   */
2680
2681                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2682                         break;
2683
2684                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2685                                         /* op1 = target JavaVM pc, val.l = constant   */
2686
2687                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2688                         break;
2689
2690                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2691                                         /* op1 = target JavaVM pc, val.l = constant   */
2692
2693                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2694                         break;
2695
2696                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2697                                         /* op1 = target JavaVM pc, val.l = constant   */
2698
2699                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2700                         break;
2701
2702                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2703                                         /* op1 = target JavaVM pc, val.l = constant   */
2704
2705                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2706                         break;
2707
2708                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2709                                         /* op1 = target JavaVM pc                     */
2710
2711                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2712                         break;
2713
2714                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2715                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2716
2717                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2718                         break;
2719
2720                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2721                                         /* op1 = target JavaVM pc                     */
2722
2723                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2724                         break;
2725
2726                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2727                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2728
2729                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2730                         break;
2731
2732                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2733                                         /* op1 = target JavaVM pc                     */
2734
2735                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2736                         break;
2737
2738                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2739                                     /* op1 = target JavaVM pc                     */
2740
2741                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2742                         break;
2743
2744                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2745                                         /* op1 = target JavaVM pc                     */
2746
2747                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2748                         break;
2749
2750                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2751                                 /* op1 = target JavaVM pc                     */
2752
2753                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2754                         break;
2755
2756                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2757                                         /* op1 = target JavaVM pc                     */
2758
2759                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2760                         break;
2761
2762                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2763                                         /* op1 = target JavaVM pc                     */
2764
2765                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2766                         break;
2767
2768                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2769                                         /* op1 = target JavaVM pc                     */
2770
2771                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2772                         break;
2773
2774                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2775                                     /* op1 = target JavaVM pc                     */
2776
2777                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2778                         break;
2779
2780                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2781
2782                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2783                         break;
2784
2785                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2786                                         /* val.i = constant                           */
2787
2788                         var_to_reg_int(s1, src, REG_ITMP1);
2789                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2790                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2791                                 if (s1 == d) {
2792                                         M_INTMOVE(s1, REG_ITMP1);
2793                                         s1 = REG_ITMP1;
2794                                 }
2795                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2796                         }
2797                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2798                         x86_64_testl_reg_reg(cd, s1, s1);
2799                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2800                         store_reg_to_var_int(iptr->dst, d);
2801                         break;
2802
2803                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2804                                         /* val.i = constant                           */
2805
2806                         var_to_reg_int(s1, src, REG_ITMP1);
2807                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2808                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2809                                 if (s1 == d) {
2810                                         M_INTMOVE(s1, REG_ITMP1);
2811                                         s1 = REG_ITMP1;
2812                                 }
2813                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2814                         }
2815                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2816                         x86_64_testl_reg_reg(cd, s1, s1);
2817                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2818                         store_reg_to_var_int(iptr->dst, d);
2819                         break;
2820
2821                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2822                                         /* val.i = constant                           */
2823
2824                         var_to_reg_int(s1, src, REG_ITMP1);
2825                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2826                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2827                                 if (s1 == d) {
2828                                         M_INTMOVE(s1, REG_ITMP1);
2829                                         s1 = REG_ITMP1;
2830                                 }
2831                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2832                         }
2833                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2834                         x86_64_testl_reg_reg(cd, s1, s1);
2835                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2836                         store_reg_to_var_int(iptr->dst, d);
2837                         break;
2838
2839                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2840                                         /* val.i = constant                           */
2841
2842                         var_to_reg_int(s1, src, REG_ITMP1);
2843                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2844                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2845                                 if (s1 == d) {
2846                                         M_INTMOVE(s1, REG_ITMP1);
2847                                         s1 = REG_ITMP1;
2848                                 }
2849                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2850                         }
2851                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2852                         x86_64_testl_reg_reg(cd, s1, s1);
2853                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2854                         store_reg_to_var_int(iptr->dst, d);
2855                         break;
2856
2857                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2858                                         /* val.i = constant                           */
2859
2860                         var_to_reg_int(s1, src, REG_ITMP1);
2861                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2862                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2863                                 if (s1 == d) {
2864                                         M_INTMOVE(s1, REG_ITMP1);
2865                                         s1 = REG_ITMP1;
2866                                 }
2867                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2868                         }
2869                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2870                         x86_64_testl_reg_reg(cd, s1, s1);
2871                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2872                         store_reg_to_var_int(iptr->dst, d);
2873                         break;
2874
2875                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2876                                         /* val.i = constant                           */
2877
2878                         var_to_reg_int(s1, src, REG_ITMP1);
2879                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2880                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2881                                 if (s1 == d) {
2882                                         M_INTMOVE(s1, REG_ITMP1);
2883                                         s1 = REG_ITMP1;
2884                                 }
2885                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2886                         }
2887                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2888                         x86_64_testl_reg_reg(cd, s1, s1);
2889                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2890                         store_reg_to_var_int(iptr->dst, d);
2891                         break;
2892
2893
2894                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2895                 case ICMD_LRETURN:
2896
2897                         var_to_reg_int(s1, src, REG_RESULT);
2898                         M_INTMOVE(s1, REG_RESULT);
2899                         goto nowperformreturn;
2900
2901                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2902
2903                         var_to_reg_int(s1, src, REG_RESULT);
2904                         M_INTMOVE(s1, REG_RESULT);
2905
2906                         if (iptr->val.a) {
2907                                 codegen_addpatchref(cd, cd->mcodeptr,
2908                                                                         PATCHER_athrow_areturn,
2909                                                                         (unresolved_class *) iptr->val.a, 0);
2910
2911                                 if (opt_showdisassemble) {
2912                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2913                                 }
2914                         }
2915                         goto nowperformreturn;
2916
2917                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2918                 case ICMD_DRETURN:
2919
2920                         var_to_reg_flt(s1, src, REG_FRESULT);
2921                         M_FLTMOVE(s1, REG_FRESULT);
2922                         goto nowperformreturn;
2923
2924                 case ICMD_RETURN:      /* ...  ==> ...                                */
2925
2926 nowperformreturn:
2927                         {
2928                         s4 i, p;
2929
2930                         p = parentargs_base;
2931
2932                         /* call trace function */
2933                         if (runverbose) {
2934                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2935
2936                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2937                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2938
2939                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2940                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2941                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2942                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2943
2944                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2945                                 x86_64_call_reg(cd, REG_ITMP1);
2946
2947                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2948                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2949
2950                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2951                         }
2952
2953 #if defined(USE_THREADS)
2954                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2955                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2956
2957                                 /* we need to save the proper return value */
2958                                 switch (iptr->opc) {
2959                                 case ICMD_IRETURN:
2960                                 case ICMD_ARETURN:
2961                                 case ICMD_LRETURN:
2962                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2963                                         break;
2964                                 case ICMD_FRETURN:
2965                                 case ICMD_DRETURN:
2966                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2967                                         break;
2968                                 }
2969
2970                                 M_MOV_IMM((ptrint) builtin_monitorexit, REG_ITMP1);
2971                                 M_CALL(REG_ITMP1);
2972
2973                                 /* and now restore the proper return value */
2974                                 switch (iptr->opc) {
2975                                 case ICMD_IRETURN:
2976                                 case ICMD_ARETURN:
2977                                 case ICMD_LRETURN:
2978                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2979                                         break;
2980                                 case ICMD_FRETURN:
2981                                 case ICMD_DRETURN:
2982                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2983                                         break;
2984                                 }
2985                         }
2986 #endif
2987
2988                         /* restore saved registers */
2989
2990                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2991                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2992                         }
2993                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
2994                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
2995                         }
2996
2997                         /* deallocate stack */
2998
2999                         if (parentargs_base)
3000                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
3001
3002                         M_RET;
3003                         }
3004                         break;
3005
3006
3007                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3008                         {
3009                                 s4 i, l, *s4ptr;
3010                                 void **tptr;
3011
3012                                 tptr = (void **) iptr->target;
3013
3014                                 s4ptr = iptr->val.a;
3015                                 l = s4ptr[1];                          /* low     */
3016                                 i = s4ptr[2];                          /* high    */
3017
3018                                 var_to_reg_int(s1, src, REG_ITMP1);
3019                                 M_INTMOVE(s1, REG_ITMP1);
3020                                 if (l != 0) {
3021                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3022                                 }
3023                                 i = i - l + 1;
3024
3025                 /* range check */
3026                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3027                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3028
3029                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3030                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3031
3032                                 /* build jump table top down and use address of lowest entry */
3033
3034                 /* s4ptr += 3 + i; */
3035                                 tptr += i;
3036
3037                                 while (--i >= 0) {
3038                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3039                                         --tptr;
3040                                 }
3041
3042                                 /* length of dataseg after last dseg_addtarget is used by load */
3043
3044                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3045                                 dseg_adddata(cd, cd->mcodeptr);
3046                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3047                                 x86_64_jmp_reg(cd, REG_ITMP1);
3048                         }
3049                         break;
3050
3051
3052                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3053                         {
3054                                 s4 i, l, val, *s4ptr;
3055                                 void **tptr;
3056
3057                                 tptr = (void **) iptr->target;
3058
3059                                 s4ptr = iptr->val.a;
3060                                 l = s4ptr[0];                          /* default  */
3061                                 i = s4ptr[1];                          /* count    */
3062
3063                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3064                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3065                                 while (--i >= 0) {
3066                                         s4ptr += 2;
3067                                         ++tptr;
3068
3069                                         val = s4ptr[0];
3070                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3071                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3072                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3073                                 }
3074
3075                                 x86_64_jmp_imm(cd, 0);
3076
3077                                 tptr = (void **) iptr->target;
3078                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3079                         }
3080                         break;
3081
3082
3083                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3084                                         /* op1 = arg count val.a = builtintable entry */
3085
3086                         bte = iptr->val.a;
3087                         md = bte->md;
3088                         goto gen_method;
3089
3090                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3091                                         /* op1 = arg count, val.a = method pointer    */
3092
3093                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3094                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3095                 case ICMD_INVOKEINTERFACE:
3096
3097                         lm = iptr->val.a;
3098
3099                         if (lm == NULL) {
3100                                 unresolved_method *um = iptr->target;
3101                                 md = um->methodref->parseddesc.md;
3102                         } else {
3103                                 md = lm->parseddesc;
3104                         }
3105
3106 gen_method:
3107                         s3 = md->paramcount;
3108
3109                         MCODECHECK((20 * s3) + 128);
3110
3111                         /* copy arguments to registers or stack location */
3112
3113                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3114                                 if (src->varkind == ARGVAR)
3115                                         continue;
3116                                 if (IS_INT_LNG_TYPE(src->type)) {
3117                                         if (!md->params[s3].inmemory) {
3118                                                 s1 = rd->argintregs[md->params[s3].regoff];
3119                                                 var_to_reg_int(d, src, s1);
3120                                                 M_INTMOVE(d, s1);
3121                                         } else {
3122                                                 var_to_reg_int(d, src, REG_ITMP1);
3123                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3124                                         }
3125
3126                                 } else {
3127                                         if (!md->params[s3].inmemory) {
3128                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3129                                                 var_to_reg_flt(d, src, s1);
3130                                                 M_FLTMOVE(d, s1);
3131                                         } else {
3132                                                 var_to_reg_flt(d, src, REG_FTMP1);
3133                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3134                                         }
3135                                 }
3136                         }
3137
3138                         switch (iptr->opc) {
3139                         case ICMD_BUILTIN:
3140                                 a = (ptrint) bte->fp;
3141                                 d = md->returntype.type;
3142
3143                                 M_MOV_IMM(a, REG_ITMP1);
3144                                 M_CALL(REG_ITMP1);
3145
3146                                 /* if op1 == true, we need to check for an exception */
3147
3148                                 if (iptr->op1 == true) {
3149                                         M_TEST(REG_RESULT);
3150                                         M_BEQ(0);
3151                                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3152                                 }
3153                                 break;
3154
3155                         case ICMD_INVOKESPECIAL:
3156                                 M_TEST(rd->argintregs[0]);
3157                                 M_BEQ(0);
3158                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3159
3160                                 /* first argument contains pointer */
3161 /*                              gen_nullptr_check(rd->argintregs[0]); */
3162
3163                                 /* access memory for hardware nullptr */
3164 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3165
3166                                 /* fall through */
3167
3168                         case ICMD_INVOKESTATIC:
3169                                 if (lm == NULL) {
3170                                         unresolved_method *um = iptr->target;
3171
3172                                         codegen_addpatchref(cd, cd->mcodeptr,
3173                                                                                 PATCHER_invokestatic_special, um, 0);
3174
3175                                         if (opt_showdisassemble) {
3176                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3177                                         }
3178
3179                                         a = 0;
3180                                         d = um->methodref->parseddesc.md->returntype.type;
3181
3182                                 } else {
3183                                         a = (ptrint) lm->stubroutine;
3184                                         d = lm->parseddesc->returntype.type;
3185                                 }
3186
3187                                 M_MOV_IMM(a, REG_ITMP2);
3188                                 M_CALL(REG_ITMP2);
3189                                 break;
3190
3191                         case ICMD_INVOKEVIRTUAL:
3192                                 gen_nullptr_check(rd->argintregs[0]);
3193
3194                                 if (lm == NULL) {
3195                                         unresolved_method *um = iptr->target;
3196
3197                                         codegen_addpatchref(cd, cd->mcodeptr,
3198                                                                                 PATCHER_invokevirtual, um, 0);
3199
3200                                         if (opt_showdisassemble) {
3201                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3202                                         }
3203
3204                                         s1 = 0;
3205                                         d = um->methodref->parseddesc.md->returntype.type;
3206
3207                                 } else {
3208                                         s1 = OFFSET(vftbl_t, table[0]) +
3209                                                 sizeof(methodptr) * lm->vftblindex;
3210                                         d = lm->parseddesc->returntype.type;
3211                                 }
3212
3213                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3214                                                                            OFFSET(java_objectheader, vftbl),
3215                                                                            REG_ITMP2);
3216                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3217                                 M_CALL(REG_ITMP1);
3218                                 break;
3219
3220                         case ICMD_INVOKEINTERFACE:
3221                                 gen_nullptr_check(rd->argintregs[0]);
3222
3223                                 if (lm == NULL) {
3224                                         unresolved_method *um = iptr->target;
3225
3226                                         codegen_addpatchref(cd, cd->mcodeptr,
3227                                                                                 PATCHER_invokeinterface, um, 0);
3228
3229                                         if (opt_showdisassemble) {
3230                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3231                                         }
3232
3233                                         s1 = 0;
3234                                         s2 = 0;
3235                                         d = um->methodref->parseddesc.md->returntype.type;
3236
3237                                 } else {
3238                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3239                                                 sizeof(methodptr) * lm->class->index;
3240
3241                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3242
3243                                         d = lm->parseddesc->returntype.type;
3244                                 }
3245
3246                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3247                                           OFFSET(java_objectheader, vftbl));
3248                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3249                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3250                                 M_CALL(REG_ITMP1);
3251                                 break;
3252                         }
3253
3254                         /* d contains return type */
3255
3256                         if (d != TYPE_VOID) {
3257                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3258                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3259                                         M_INTMOVE(REG_RESULT, s1);
3260                                         store_reg_to_var_int(iptr->dst, s1);
3261                                 } else {
3262                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3263                                         M_FLTMOVE(REG_FRESULT, s1);
3264                                         store_reg_to_var_flt(iptr->dst, s1);
3265                                 }
3266                         }
3267                         break;
3268
3269
3270                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3271
3272                                       /* op1:   0 == array, 1 == class                */
3273                                       /* val.a: (classinfo *) superclass              */
3274
3275                         /*  superclass is an interface:
3276                          *
3277                          *  OK if ((sub == NULL) ||
3278                          *         (sub->vftbl->interfacetablelength > super->index) &&
3279                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3280                          *
3281                          *  superclass is a class:
3282                          *
3283                          *  OK if ((sub == NULL) || (0
3284                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3285                          *         super->vftbl->diffval));
3286                          */
3287
3288                         if (iptr->op1 == 1) {
3289                                 /* object type cast-check */
3290
3291                                 classinfo *super;
3292                                 vftbl_t   *supervftbl;
3293                                 s4         superindex;
3294
3295                                 super = (classinfo *) iptr->val.a;
3296
3297                                 if (!super) {
3298                                         superindex = 0;
3299                                         supervftbl = NULL;
3300
3301                                 } else {
3302                                         superindex = super->index;
3303                                         supervftbl = super->vftbl;
3304                                 }
3305
3306 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3307                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3308 #endif
3309                                 var_to_reg_int(s1, src, REG_ITMP1);
3310
3311                                 /* calculate interface checkcast code size */
3312
3313                                 s2 = 3; /* mov_membase_reg */
3314                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3315
3316                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3317                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3318                                         3 /* test */ + 6 /* jcc */;
3319
3320                                 if (!super)
3321                                         s2 += (opt_showdisassemble ? 5 : 0);
3322
3323                                 /* calculate class checkcast code size */
3324
3325                                 s3 = 3; /* mov_membase_reg */
3326                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3327                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3328
3329 #if 0
3330                                 if (s1 != REG_ITMP1) {
3331                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3332                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3333                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3334                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3335                                         a += 3;    /* sub */
3336
3337                                 } else
3338 #endif
3339                                         {
3340                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3341                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3342                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3343                                         }
3344
3345                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3346
3347                                 if (!super)
3348                                         s3 += (opt_showdisassemble ? 5 : 0);
3349
3350                                 /* if class is not resolved, check which code to call */
3351
3352                                 if (!super) {
3353                                         M_TEST(s1);
3354                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3355
3356                                         codegen_addpatchref(cd, cd->mcodeptr,
3357                                                                                 PATCHER_checkcast_instanceof_flags,
3358                                                                                 (constant_classref *) iptr->target, 0);
3359
3360                                         if (opt_showdisassemble) {
3361                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3362                                         }
3363
3364                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3365                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3366                                         M_BEQ(s2 + 5);
3367                                 }
3368
3369                                 /* interface checkcast code */
3370
3371                                 if (!super || (super->flags & ACC_INTERFACE)) {
3372                                         if (super) {
3373                                                 M_TEST(s1);
3374                                                 M_BEQ(s2);
3375                                         }
3376
3377                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3378
3379                                         if (!super) {
3380                                                 codegen_addpatchref(cd, cd->mcodeptr,
3381                                                                                         PATCHER_checkcast_instanceof_interface,
3382                                                                                         (constant_classref *) iptr->target, 0);
3383
3384                                                 if (opt_showdisassemble) {
3385                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3386                                                 }
3387                                         }
3388
3389                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3390                                                                                           OFFSET(vftbl_t, interfacetablelength),
3391                                                                                           REG_ITMP3);
3392                                         /* XXX TWISTI: should this be int arithmetic? */
3393                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3394                                         M_TEST(REG_ITMP3);
3395                                         M_BLE(0);
3396                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3397                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3398                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3399                                                                                          superindex * sizeof(methodptr*),
3400                                                                                          REG_ITMP3);
3401                                         M_TEST(REG_ITMP3);
3402                                         M_BEQ(0);
3403                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3404
3405                                         if (!super)
3406                                                 M_JMP_IMM(s3);
3407                                 }
3408
3409                                 /* class checkcast code */
3410
3411                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3412                                         if (super) {
3413                                                 M_TEST(s1);
3414                                                 M_BEQ(s3);
3415                                         }
3416
3417                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3418
3419                                         if (!super) {
3420                                                 codegen_addpatchref(cd, cd->mcodeptr,
3421                                                                                         PATCHER_checkcast_class,
3422                                                                                         (constant_classref *) iptr->target,
3423                                                                                         0);
3424
3425                                                 if (opt_showdisassemble) {
3426                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3427                                                 }
3428                                         }
3429
3430                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3431 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3432                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3433 #endif
3434                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3435                                                                                           OFFSET(vftbl_t, baseval),
3436                                                                                           REG_ITMP2);
3437                                         /*                                      if (s1 != REG_ITMP1) { */
3438                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3439                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3440                                         /*                                                                                              REG_ITMP1); */
3441                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3442                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3443                                         /*                                                                                              REG_ITMP3); */
3444                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3445                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3446                                         /*  #endif */
3447                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3448
3449                                         /*                                      } else { */
3450                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3451                                                                                           OFFSET(vftbl_t, baseval),
3452                                                                                           REG_ITMP3);
3453                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3454                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3455                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3456                                         /*                                      } */
3457 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3458                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3459 #endif
3460                                         M_CMP(REG_ITMP3, REG_ITMP2);
3461                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3462                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3463                                 }
3464                                 d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3465
3466                         } else {
3467                                 /* array type cast-check */
3468
3469                                 var_to_reg_int(s1, src, REG_ITMP1);
3470                                 M_INTMOVE(s1, rd->argintregs[0]);
3471
3472                                 if (iptr->val.a == NULL) {
3473                                         codegen_addpatchref(cd, cd->mcodeptr,
3474                                                                                 PATCHER_builtin_arraycheckcast,
3475                                                                                 (constant_classref *) iptr->target, 0);
3476
3477                                         if (opt_showdisassemble) {
3478                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3479                                         }
3480
3481                                         a = 0;
3482
3483                                 } else {
3484                                         a = (ptrint) BUILTIN_arraycheckcast;
3485                                 }
3486
3487                                 M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3488                                 M_MOV_IMM((ptrint) a, REG_ITMP1);
3489                                 M_CALL(REG_ITMP1);
3490                                 M_TEST(REG_RESULT);
3491                                 M_BEQ(0);
3492                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3493
3494                                 var_to_reg_int(s1, src, REG_ITMP1);
3495                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3496                         }
3497                         M_INTMOVE(s1, d);
3498                         store_reg_to_var_int(iptr->dst, d);
3499                         break;
3500
3501                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3502
3503                                       /* op1:   0 == array, 1 == class                */
3504                                       /* val.a: (classinfo *) superclass              */
3505
3506                         /*  superclass is an interface:
3507                          *
3508                          *  return (sub != NULL) &&
3509                          *         (sub->vftbl->interfacetablelength > super->index) &&
3510                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3511                          *
3512                          *  superclass is a class:
3513                          *
3514                          *  return ((sub != NULL) && (0
3515                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3516                          *          super->vftbl->diffvall));
3517                          */
3518
3519                         {
3520                         classinfo *super;
3521                         vftbl_t   *supervftbl;
3522                         s4         superindex;
3523
3524                         super = (classinfo *) iptr->val.a;
3525
3526                         if (!super) {
3527                                 superindex = 0;
3528                                 supervftbl = NULL;
3529
3530                         } else {
3531                                 superindex = super->index;
3532                                 supervftbl = super->vftbl;
3533                         }
3534
3535 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3536             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3537 #endif
3538
3539                         var_to_reg_int(s1, src, REG_ITMP1);
3540                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3541                         if (s1 == d) {
3542                                 M_INTMOVE(s1, REG_ITMP1);
3543                                 s1 = REG_ITMP1;
3544                         }
3545
3546                         /* calculate interface instanceof code size */
3547
3548                         s2 = 3; /* mov_membase_reg */
3549                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3550                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3551                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3552                                 3 /* test */ + 4 /* setcc */;
3553
3554                         if (!super)
3555                                 s2 += (opt_showdisassemble ? 5 : 0);
3556
3557                         /* calculate class instanceof code size */
3558
3559                         s3 = 3; /* mov_membase_reg */
3560                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3561                         s3 += 10; /* mov_imm_reg */
3562                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3563                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3564                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3565                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3566                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3567                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3568                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3569
3570                         if (!super)
3571                                 s3 += (opt_showdisassemble ? 5 : 0);
3572
3573                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3574
3575                         /* if class is not resolved, check which code to call */
3576
3577                         if (!super) {
3578                                 x86_64_test_reg_reg(cd, s1, s1);
3579                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3580                                                                                          7 + 6 + s2 + 5 + s3));
3581
3582                                 codegen_addpatchref(cd, cd->mcodeptr,
3583                                                                         PATCHER_checkcast_instanceof_flags,
3584                                                                         (constant_classref *) iptr->target, 0);
3585
3586                                 if (opt_showdisassemble) {
3587                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3588                                 }
3589
3590                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3591                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3592                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3593                         }
3594
3595                         /* interface instanceof code */
3596
3597                         if (!super || (super->flags & ACC_INTERFACE)) {
3598                                 if (super) {
3599                                         x86_64_test_reg_reg(cd, s1, s1);
3600                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3601                                 }
3602
3603                                 x86_64_mov_membase_reg(cd, s1,
3604                                                                            OFFSET(java_objectheader, vftbl),
3605                                                                            REG_ITMP1);
3606                                 if (!super) {
3607                                         codegen_addpatchref(cd, cd->mcodeptr,
3608                                                                                 PATCHER_checkcast_instanceof_interface,
3609                                                                                 (constant_classref *) iptr->target, 0);
3610
3611                                         if (opt_showdisassemble) {
3612                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3613                                         }
3614                                 }
3615
3616                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3617                                                                                   OFFSET(vftbl_t, interfacetablelength),
3618                                                                                   REG_ITMP3);
3619                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3620                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3621
3622                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3623
3624                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3625                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3626                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3627                                                                                  superindex * sizeof(methodptr*),
3628                                                                                  REG_ITMP1);
3629                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3630                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3631
3632                                 if (!super)
3633                                         x86_64_jmp_imm(cd, s3);
3634                         }
3635
3636                         /* class instanceof code */
3637
3638                         if (!super || !(super->flags & ACC_INTERFACE)) {
3639                                 if (super) {
3640                                         x86_64_test_reg_reg(cd, s1, s1);
3641                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3642                                 }
3643
3644                                 x86_64_mov_membase_reg(cd, s1,
3645                                                                            OFFSET(java_objectheader, vftbl),
3646                                                                            REG_ITMP1);
3647
3648                                 if (!super) {
3649                                         codegen_addpatchref(cd, cd->mcodeptr,
3650                                                                                 PATCHER_instanceof_class,
3651                                                                                 (constant_classref *) iptr->target, 0);
3652
3653                                         if (opt_showdisassemble) {
3654                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3655                                         }
3656                                 }
3657
3658                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3659 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3660                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3661 #endif
3662                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3663                                                                                 OFFSET(vftbl_t, baseval),
3664                                                                                 REG_ITMP1);
3665                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3666                                                                                 OFFSET(vftbl_t, diffval),
3667                                                                                 REG_ITMP3);
3668                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3669                                                                                 OFFSET(vftbl_t, baseval),
3670                                                                                 REG_ITMP2);
3671 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3672                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3673 #endif
3674                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3675                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3676                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3677                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3678                         }
3679                         store_reg_to_var_int(iptr->dst, d);
3680                         }
3681                         break;
3682
3683                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3684                                          /* op1 = dimension, val.a = array descriptor */
3685
3686                         /* check for negative sizes and copy sizes to stack if necessary  */
3687
3688                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3689
3690                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3691                                 /* copy SAVEDVAR sizes to stack */
3692
3693                                 if (src->varkind != ARGVAR) {
3694                                         var_to_reg_int(s2, src, REG_ITMP1);
3695                                         M_LST(s2, REG_SP, s1 * 8);
3696                                 }
3697                         }
3698
3699                         /* is a patcher function set? */
3700
3701                         if (iptr->target) {
3702                                 codegen_addpatchref(cd, cd->mcodeptr,
3703                                                                         (functionptr) (ptrint) iptr->target,
3704                                                                         iptr->val.a, 0);
3705
3706                                 if (opt_showdisassemble) {
3707                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3708                                 }
3709
3710                                 a = 0;
3711
3712                         } else {
3713                                 a = (ptrint) iptr->val.a;
3714                         }
3715
3716                         /* a0 = dimension count */
3717
3718                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3719
3720                         /* a1 = arrayvftbl */
3721
3722                         M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3723
3724                         /* a2 = pointer to dimensions = stack pointer */
3725
3726                         M_MOV(REG_SP, rd->argintregs[2]);
3727
3728                         M_MOV_IMM((ptrint) BUILTIN_multianewarray, REG_ITMP1);
3729                         M_CALL(REG_ITMP1);
3730
3731                         /* check for exception before result assignment */
3732
3733                         M_TEST(REG_RESULT);
3734                         M_BEQ(0);
3735                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3736
3737                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3738                         M_INTMOVE(REG_RESULT, s1);
3739                         store_reg_to_var_int(iptr->dst, s1);
3740                         break;
3741
3742                 default:
3743                         throw_cacao_exception_exit(string_java_lang_InternalError,
3744                                                                            "Unknown ICMD %d", iptr->opc);
3745         } /* switch */
3746
3747         } /* for instruction */
3748
3749         /* copy values to interface registers */
3750
3751         src = bptr->outstack;
3752         len = bptr->outdepth;
3753         MCODECHECK(512);
3754 #ifdef LSRA
3755         if (!opt_lsra)
3756 #endif
3757         while (src) {
3758                 len--;
3759                 if ((src->varkind != STACKVAR)) {
3760                         s2 = src->type;
3761                         if (IS_FLT_DBL_TYPE(s2)) {
3762                                 var_to_reg_flt(s1, src, REG_FTMP1);
3763                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3764                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3765
3766                                 } else {
3767                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3768                                 }
3769
3770                         } else {
3771                                 var_to_reg_int(s1, src, REG_ITMP1);
3772                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3773                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3774
3775                                 } else {
3776                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3777                                 }
3778                         }
3779                 }
3780                 src = src->prev;
3781         }
3782
3783         /* At the end of a basic block we may have to append some nops,
3784            because the patcher stub calling code might be longer than the
3785            actual instruction. So codepatching does not change the
3786            following block unintentionally. */
3787
3788         if (cd->mcodeptr < cd->lastmcodeptr) {
3789                 while (cd->mcodeptr < cd->lastmcodeptr) {
3790                         M_NOP;
3791                 }
3792         }
3793
3794         } /* if (bptr -> flags >= BBREACHED) */
3795         } /* for basic block */
3796
3797         codegen_createlinenumbertable(cd);
3798
3799         {
3800
3801         u1        *xcodeptr;
3802         branchref *bref;
3803
3804         /* generate ArithmeticException stubs */
3805
3806         xcodeptr = NULL;
3807
3808         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3809                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3810                                   bref->branchpos,
3811                                                   cd->mcodeptr - cd->mcodebase);
3812
3813                 MCODECHECK(512);
3814
3815                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3816                 dseg_adddata(cd, cd->mcodeptr);
3817                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3818
3819                 if (xcodeptr != NULL) {
3820                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3821
3822                 } else {
3823                         xcodeptr = cd->mcodeptr;
3824
3825                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3826                         M_MOV(REG_SP, rd->argintregs[1]);
3827                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3828                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3829
3830                         M_ASUB_IMM(2 * 8, REG_SP);
3831                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3832
3833                         M_MOV_IMM((ptrint) stacktrace_inline_arithmeticexception,
3834                                           REG_ITMP3);
3835                         M_CALL(REG_ITMP3);
3836
3837                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3838                         M_AADD_IMM(2 * 8, REG_SP);
3839
3840                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3841                         M_JMP(REG_ITMP3);
3842                 }
3843         }
3844
3845         /* generate ArrayIndexOutOfBoundsException stubs */
3846
3847         xcodeptr = NULL;
3848
3849         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3850                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3851                                   bref->branchpos,
3852                                                   cd->mcodeptr - cd->mcodebase);
3853
3854                 MCODECHECK(512);
3855
3856                 /* move index register into REG_ITMP1 */
3857
3858                 M_MOV(bref->reg, REG_ITMP1);                              /* 3 bytes  */
3859
3860                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3861                 dseg_adddata(cd, cd->mcodeptr);
3862                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3863
3864                 if (xcodeptr != NULL) {
3865                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3866
3867                 } else {
3868                         xcodeptr = cd->mcodeptr;
3869
3870                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3871                         M_MOV(REG_SP, rd->argintregs[1]);
3872                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3873                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3874                         M_MOV(REG_ITMP1, rd->argintregs[4]);
3875
3876                         M_ASUB_IMM(2 * 8, REG_SP);
3877                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3878
3879                         M_MOV_IMM((ptrint) stacktrace_inline_arrayindexoutofboundsexception,
3880                                           REG_ITMP3);
3881                         M_CALL(REG_ITMP3);
3882
3883                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3884                         M_AADD_IMM(2 * 8, REG_SP);
3885
3886                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3887                         M_JMP(REG_ITMP3);
3888                 }
3889         }
3890
3891         /* generate ArrayStoreException stubs */
3892
3893         xcodeptr = NULL;
3894
3895         for (bref = cd->xstorerefs; bref != NULL; bref = bref->next) {
3896                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3897                                   bref->branchpos,
3898                                                   cd->mcodeptr - cd->mcodebase);
3899
3900                 MCODECHECK(512);
3901
3902                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3903                 dseg_adddata(cd, cd->mcodeptr);
3904                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3905
3906                 if (xcodeptr != NULL) {
3907                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3908
3909                 } else {
3910                         xcodeptr = cd->mcodeptr;
3911
3912                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3913                         M_MOV(REG_SP, rd->argintregs[1]);
3914                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3915                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3916
3917                         M_ASUB_IMM(2 * 8, REG_SP);
3918                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3919
3920                         M_MOV_IMM((ptrint) stacktrace_inline_arraystoreexception,
3921                                           REG_ITMP3);
3922                         M_CALL(REG_ITMP3);
3923
3924                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3925                         M_AADD_IMM(2 * 8, REG_SP);
3926
3927                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3928                         M_JMP(REG_ITMP3);
3929                 }
3930         }
3931
3932         /* generate ClassCastException stubs */
3933
3934         xcodeptr = NULL;
3935
3936         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3937                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3938                                   bref->branchpos,
3939                                                   cd->mcodeptr - cd->mcodebase);
3940
3941                 MCODECHECK(512);
3942
3943                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3944                 dseg_adddata(cd, cd->mcodeptr);
3945                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3946
3947                 if (xcodeptr != NULL) {
3948                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3949
3950                 } else {
3951                         xcodeptr = cd->mcodeptr;
3952
3953                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3954                         M_MOV(REG_SP, rd->argintregs[1]);
3955                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3956                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3957
3958                         M_ASUB_IMM(2 * 8, REG_SP);
3959                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3960
3961                         M_MOV_IMM((ptrint) stacktrace_inline_classcastexception, REG_ITMP3);
3962                         M_CALL(REG_ITMP3);
3963
3964                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3965                         M_AADD_IMM(2 * 8, REG_SP);
3966
3967                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3968                         M_JMP(REG_ITMP3);
3969                 }
3970         }
3971
3972         /* generate NullpointerException stubs */
3973
3974         xcodeptr = NULL;
3975
3976         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3977                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3978                                   bref->branchpos,
3979                                                   cd->mcodeptr - cd->mcodebase);
3980
3981                 MCODECHECK(512);
3982
3983                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3984                 dseg_adddata(cd, cd->mcodeptr);
3985                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3986
3987                 if (xcodeptr != NULL) {
3988                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3989
3990                 } else {
3991                         xcodeptr = cd->mcodeptr;
3992
3993                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3994                         M_MOV(REG_SP, rd->argintregs[1]);
3995                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3996                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3997
3998                         M_ASUB_IMM(2 * 8, REG_SP);
3999                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4000
4001                         M_MOV_IMM((ptrint) stacktrace_inline_nullpointerexception,
4002                                           REG_ITMP3);
4003                         M_CALL(REG_ITMP3);
4004
4005                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4006                         M_AADD_IMM(2 * 8, REG_SP);
4007
4008                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4009                         M_JMP(REG_ITMP3);
4010                 }
4011         }
4012
4013         /* generate exception check stubs */
4014
4015         xcodeptr = NULL;
4016
4017         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
4018                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4019                                   bref->branchpos,
4020                                                   cd->mcodeptr - cd->mcodebase);
4021
4022                 MCODECHECK(512);
4023
4024                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4025                 dseg_adddata(cd, cd->mcodeptr);
4026                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4027
4028                 if (xcodeptr != NULL) {
4029                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4030
4031                 } else {
4032                         xcodeptr = cd->mcodeptr;
4033
4034                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4035                         M_MOV(REG_SP, rd->argintregs[1]);
4036                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4037                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4038
4039                         M_ASUB_IMM(2 * 8, REG_SP);
4040                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4041
4042                         M_MOV_IMM((ptrint) stacktrace_inline_fillInStackTrace, REG_ITMP3);
4043                         M_CALL(REG_ITMP3);
4044
4045                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4046                         M_AADD_IMM(2 * 8, REG_SP);
4047
4048                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4049                         M_JMP(REG_ITMP3);
4050                 }
4051         }
4052
4053         /* generate code patching stub call code */
4054
4055         {
4056                 patchref    *pref;
4057                 codegendata *tmpcd;
4058                 ptrint       mcode;
4059
4060                 tmpcd = DNEW(codegendata);
4061
4062                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4063                         /* check size of code segment */
4064
4065                         MCODECHECK(512);
4066
4067                         /* Get machine code which is patched back in later. A             */
4068                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4069
4070                         xcodeptr = cd->mcodebase + pref->branchpos;
4071                         mcode = *((ptrint *) xcodeptr);
4072
4073                         /* patch in `call rel32' to call the following code               */
4074
4075                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4076                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4077
4078                         /* move pointer to java_objectheader onto stack */
4079
4080 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4081                         /* create a virtual java_objectheader */
4082
4083                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4084                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4085
4086                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4087                         M_PUSH(REG_ITMP3);
4088 #else
4089                         M_PUSH_IMM(0);
4090 #endif
4091
4092                         /* move machine code bytes and classinfo pointer into registers */
4093
4094                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4095                         M_PUSH(REG_ITMP3);
4096                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4097                         M_PUSH(REG_ITMP3);
4098                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4099                         M_PUSH(REG_ITMP3);
4100
4101                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4102                         M_PUSH(REG_ITMP3);
4103
4104                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4105                         M_JMP(REG_ITMP3);
4106                 }
4107         }
4108         }
4109
4110         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4111 }
4112
4113
4114 /* createcompilerstub **********************************************************
4115
4116    Creates a stub routine which calls the compiler.
4117
4118 *******************************************************************************/
4119
4120 #define COMPILERSTUB_SIZE    23
4121
4122 functionptr createcompilerstub(methodinfo *m)
4123 {
4124         u1          *s;                     /* memory to hold the stub            */
4125         codegendata *cd;
4126         s4           dumpsize;
4127
4128         s = CNEW(u1, COMPILERSTUB_SIZE);
4129
4130         /* mark start of dump memory area */
4131
4132         dumpsize = dump_size();
4133
4134         cd = DNEW(codegendata);
4135         cd->mcodeptr = s;
4136
4137         /* code for the stub */
4138
4139         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1); /* pass method to compiler */
4140         x86_64_mov_imm_reg(cd, (ptrint) asm_call_jit_compiler, REG_ITMP3);
4141         x86_64_jmp_reg(cd, REG_ITMP3);
4142
4143 #if defined(STATISTICS)
4144         if (opt_stat)
4145                 count_cstub_len += COMPILERSTUB_SIZE;
4146 #endif
4147
4148         /* release dump area */
4149
4150         dump_release(dumpsize);
4151
4152         return (functionptr) (ptrint) s;
4153 }
4154
4155
4156 /* createnativestub ************************************************************
4157
4158    Creates a stub routine which calls a native method.
4159
4160 *******************************************************************************/
4161
4162 functionptr createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4163                                                          registerdata *rd, methoddesc *nmd)
4164 {
4165         methoddesc *md;
4166         s4          stackframesize;         /* size of stackframe if needed       */
4167         s4          nativeparams;
4168         s4          i, j;                   /* count variables                    */
4169         s4          t;
4170         s4          s1, s2;
4171
4172         /* initialize variables */
4173
4174         md = m->parseddesc;
4175         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4176
4177
4178         /* calculate stack frame size */
4179
4180         stackframesize =
4181                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4182                 sizeof(localref_table) / SIZEOF_VOID_P +
4183                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4184                 nmd->memuse;
4185
4186         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4187                 stackframesize++;
4188
4189
4190         /* create method header */
4191
4192         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4193         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4194         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4195         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4196         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4197         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4198         (void) dseg_addlinenumbertablesize(cd);
4199         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4200
4201
4202         /* initialize mcode variables */
4203
4204         cd->mcodeptr = (u1 *) cd->mcodebase;
4205         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4206
4207
4208         /* generate stub code */
4209
4210         M_ASUB_IMM(stackframesize * 8, REG_SP);
4211
4212         if (runverbose) {
4213                 /* save integer and float argument registers */
4214
4215                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4216                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4217                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4218
4219                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4220                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4221                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4222
4223                 /* show integer hex code for float arguments */
4224
4225                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++) {
4226                         /* if the paramtype is a float, we have to right shift all        */
4227                         /* following integer registers                                    */
4228
4229                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4230                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4231                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4232
4233                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4234                                 j++;
4235                         }
4236                 }
4237
4238                 x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP1);
4239                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, 0 * 8);
4240                 x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
4241                 x86_64_call_reg(cd, REG_ITMP1);
4242
4243                 /* restore integer and float argument registers */
4244
4245                 for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4246                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4247                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4248
4249                 for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4250                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4251                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4252         }
4253
4254
4255         /* get function address (this must happen before the stackframeinfo) */
4256
4257 #if !defined(ENABLE_STATICVM)
4258         if (f == NULL) {
4259                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4260
4261                 if (opt_showdisassemble) {
4262                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4263                 }
4264         }
4265 #endif
4266
4267         M_MOV_IMM((ptrint) f, REG_ITMP3);
4268
4269
4270         /* save integer and float argument registers */
4271
4272         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4273                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4274                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4275
4276         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4277                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4278                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4279
4280         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4281
4282         /* create dynamic stack info */
4283
4284         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4285         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4286         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4287         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4288         M_MOV_IMM((ptrint) codegen_start_native_call, REG_ITMP1);
4289         M_CALL(REG_ITMP1);
4290
4291         STATS({
4292                 x86_64_mov_imm_reg(cd, (ptrint) nativeinvokation, REG_ITMP1);
4293                 x86_64_call_reg(cd, REG_ITMP1);
4294         })
4295
4296         /* restore integer and float argument registers */
4297
4298         for (i = 0, j = 0; i < md->paramcount && i < INT_ARG_CNT; i++)
4299                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4300                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4301
4302         for (i = 0, j = 0; i < md->paramcount && i < FLT_ARG_CNT; i++)
4303                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4304                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4305
4306         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4307
4308
4309         /* copy or spill arguments to new locations */
4310
4311         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4312                 t = md->paramtypes[i].type;
4313
4314                 if (IS_INT_LNG_TYPE(t)) {
4315                         if (!md->params[i].inmemory) {
4316                                 s1 = rd->argintregs[md->params[i].regoff];
4317
4318                                 if (!nmd->params[j].inmemory) {
4319                                         s2 = rd->argintregs[nmd->params[j].regoff];
4320                                         M_INTMOVE(s1, s2);
4321
4322                                 } else {
4323                                         s2 = nmd->params[j].regoff;
4324                                         M_LST(s1, REG_SP, s2 * 8);
4325                                 }
4326
4327                         } else {
4328                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4329                                 s2 = nmd->params[j].regoff;
4330                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4331                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4332                         }
4333
4334                 } else {
4335                         /* We only copy spilled float arguments, as the float argument    */
4336                         /* registers keep unchanged.                                      */
4337
4338                         if (md->params[i].inmemory) {
4339                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4340                                 s2 = nmd->params[j].regoff;
4341                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4342                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4343                         }
4344                 }
4345         }
4346
4347         /* put class into second argument register */
4348
4349         if (m->flags & ACC_STATIC)
4350                 M_MOV_IMM((ptrint) m->class, rd->argintregs[1]);
4351
4352         /* put env into first argument register */
4353
4354         M_MOV_IMM((ptrint) &env, rd->argintregs[0]);
4355
4356         /* do the native function call */
4357
4358         M_CALL(REG_ITMP3);
4359
4360         /* save return value */
4361
4362         if (IS_INT_LNG_TYPE(md->returntype.type))
4363                 M_LST(REG_RESULT, REG_SP, 0 * 8);
4364         else
4365                 M_DST(REG_FRESULT, REG_SP, 0 * 8);
4366
4367         /* remove native stackframe info */
4368
4369         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4370         M_MOV_IMM((ptrint) codegen_finish_native_call, REG_ITMP1);
4371         M_CALL(REG_ITMP1);
4372
4373         /* generate call trace */
4374
4375         if (runverbose) {
4376                 /* just restore the value we need, don't care about the other */
4377
4378                 if (IS_INT_LNG_TYPE(md->returntype.type))
4379                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4380                 else
4381                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4382
4383                 M_MOV_IMM((ptrint) m, rd->argintregs[0]);
4384                 M_MOV(REG_RESULT, rd->argintregs[1]);
4385                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4386                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4387
4388                 M_MOV_IMM((ptrint) builtin_displaymethodstop, REG_ITMP1);
4389                 M_CALL(REG_ITMP1);
4390         }
4391
4392         /* check for exception */
4393
4394 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4395         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4396         M_CALL(REG_ITMP3);
4397 #else
4398         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_RESULT);
4399 #endif
4400         M_ALD(REG_ITMP3, REG_RESULT, 0);
4401
4402
4403         /* restore return value */
4404
4405         if (IS_INT_LNG_TYPE(md->returntype.type))
4406                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4407         else
4408                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4409
4410         /* test for exception */
4411
4412         M_TEST(REG_ITMP3);
4413         M_BNE(7 + 1);
4414
4415         /* remove stackframe */
4416
4417         M_AADD_IMM(stackframesize * 8, REG_SP);
4418         M_RET;
4419
4420
4421         /* handle exception */
4422
4423 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4424         M_LST(REG_ITMP3, REG_SP, 0 * 8);
4425         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4426         M_CALL(REG_ITMP3);
4427         x86_64_mov_imm_membase(cd, 0, REG_RESULT, 0);
4428         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4429 #else
4430         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4431         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_ITMP3);
4432         M_XOR(REG_ITMP2, REG_ITMP2);
4433         M_AST(REG_ITMP2, REG_ITMP3, 0);                /* clear exception pointer */
4434 #endif
4435
4436         /* remove stackframe */
4437
4438         M_AADD_IMM(stackframesize * 8, REG_SP);
4439
4440         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4441         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4442
4443         M_MOV_IMM((ptrint) asm_handle_nat_exception, REG_ITMP3);
4444         M_JMP(REG_ITMP3);
4445
4446
4447         /* process patcher calls **************************************************/
4448
4449         {
4450                 u1          *xcodeptr;
4451                 patchref    *pref;
4452                 codegendata *tmpcd;
4453                 ptrint       mcode;
4454 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4455                 s4           disp;
4456 #endif
4457
4458                 tmpcd = DNEW(codegendata);
4459
4460                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4461                         /* Get machine code which is patched back in later. A             */
4462                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4463
4464                         xcodeptr = cd->mcodebase + pref->branchpos;
4465                         mcode = *((ptrint *) xcodeptr);
4466
4467                         /* patch in `call rel32' to call the following code               */
4468
4469                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4470                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4471
4472                         /* move pointer to java_objectheader onto stack */
4473
4474 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4475                         /* create a virtual java_objectheader */
4476
4477                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4478                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4479
4480                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4481                         M_PUSH(REG_ITMP3);
4482 #else
4483                         M_PUSH_IMM(0);
4484 #endif
4485
4486                         /* move machine code bytes and classinfo pointer into registers */
4487
4488                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4489                         M_PUSH(REG_ITMP3);
4490                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4491                         M_PUSH(REG_ITMP3);
4492                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4493                         M_PUSH(REG_ITMP3);
4494
4495                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4496                         M_PUSH(REG_ITMP3);
4497
4498                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4499                         M_JMP(REG_ITMP3);
4500                 }
4501         }
4502
4503         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4504
4505         return m->entrypoint;
4506 }
4507
4508
4509 /*
4510  * These are local overrides for various environment variables in Emacs.
4511  * Please do not remove this and leave it at the end of the file, where
4512  * Emacs will automagically detect them.
4513  * ---------------------------------------------------------------------
4514  * Local variables:
4515  * mode: c
4516  * indent-tabs-mode: t
4517  * c-basic-offset: 4
4518  * tab-width: 4
4519  * End:
4520  */