src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005 R. Grafl, A. Krall, C. Kruegel, C. Oates,
   4    R. Obermaisser, M. Platter, M. Probst, S. Ring, E. Steiner,
   5    C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich, J. Wenninger,
   6    Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  23    02111-1307, USA.
  24
  25    Contact: cacao@complang.tuwien.ac.at
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31
  32    $Id: codegen.c 4173 2006-01-12 22:38:37Z twisti $
  33
  34 */
  35
  36
  37 #include "config.h"
  38
  39 #include <stdio.h>
  40
  41 #include "vm/types.h"
  42
  43 #include "md.h"
  44 #include "md-abi.h"
  45
  46 #include "vm/jit/x86_64/arch.h"
  47 #include "vm/jit/x86_64/codegen.h"
  48 #include "vm/jit/x86_64/emitfuncs.h"
  49
  50 #include "cacao/cacao.h"
  51 #include "native/native.h"
  52 #include "vm/builtin.h"
  53 #include "vm/exceptions.h"
  54 #include "vm/global.h"
  55 #include "vm/loader.h"
  56 #include "vm/options.h"
  57 #include "vm/statistics.h"
  58 #include "vm/stringlocal.h"
  59 #include "vm/jit/asmpart.h"
  60 #include "vm/jit/codegen-common.h"
  61 #include "vm/jit/dseg.h"
  62 #include "vm/jit/jit.h"
  63 #include "vm/jit/methodheader.h"
  64 #include "vm/jit/parse.h"
  65 #include "vm/jit/patcher.h"
  66 #include "vm/jit/reg.h"
  67
  68 #if defined(ENABLE_LSRA)
  69 # include "vm/jit/allocator/lsra.h"
  70 #endif
  71
  72
  73
  74
  75 /* codegen *********************************************************************
  76
  77    Generates machine code.
  78
  79 *******************************************************************************/
  80
  81 bool codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  82 {
  83         s4                  len, s1, s2, s3, d, disp;
  84         u2                  currentline;
  85         ptrint              a;
  86         s4                  parentargs_base;
  87         stackptr            src;
  88         varinfo            *var;
  89         basicblock         *bptr;
  90         instruction        *iptr;
  91         exceptiontable     *ex;
  92         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  93         builtintable_entry *bte;
  94         methoddesc         *md;
  95
  96         /* prevent compiler warnings */
  97
  98         d = 0;
  99         lm = NULL;
 100         bte = NULL;
 101
 102         {
 103         s4 i, p, t, l;
 104         s4 savedregs_num;
 105
 106         savedregs_num = 0;
 107
 108         /* space to save used callee saved registers */
 109
 110         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 111         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 112
 113         parentargs_base = rd->memuse + savedregs_num;
 114
 115 #if defined(USE_THREADS)
 116         /* space to save argument of monitor_enter */
 117
 118         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 119                 parentargs_base++;
 120 #endif
 121
 122     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 123         /* code e.g. libc or jni (alignment problems with movaps).                */
 124
 125         if (!m->isleafmethod || runverbose)
 126                 parentargs_base |= 0x1;
 127
 128         /* create method header */
 129
 130         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 131         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 132
 133 #if defined(USE_THREADS)
 134         /* IsSync contains the offset relative to the stack pointer for the
 135            argument of monitor_exit used in the exception handler. Since the
 136            offset could be zero and give a wrong meaning of the flag it is
 137            offset by one.
 138         */
 139
 140         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 141                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 142         else
 143 #endif
 144                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 145
 146         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 147         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 148         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 149
 150         (void) dseg_addlinenumbertablesize(cd);
 151
 152         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 153
 154         /* create exception table */
 155
 156         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 157                 dseg_addtarget(cd, ex->start);
 158                 dseg_addtarget(cd, ex->end);
 159                 dseg_addtarget(cd, ex->handler);
 160                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 161         }
 162
 163         /* initialize mcode variables */
 164
 165         cd->mcodeptr = (u1 *) cd->mcodebase;
 166         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 167         MCODECHECK(512);
 168
 169         /* initialize the last patcher pointer */
 170
 171         cd->lastmcodeptr = cd->mcodeptr;
 172
 173         /* create stack frame (if necessary) */
 174
 175         if (parentargs_base)
 176                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 177
 178         /* save used callee saved registers */
 179
 180         p = parentargs_base;
 181         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 182                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 183         }
 184         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 185                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 186         }
 187
 188         /* take arguments out of register or stack frame */
 189
 190         md = m->parseddesc;
 191
 192         for (p = 0, l = 0; p < md->paramcount; p++) {
 193                 t = md->paramtypes[p].type;
 194                 var = &(rd->locals[l][t]);
 195                 l++;
 196                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 197                         l++;
 198                 if (var->type < 0)
 199                         continue;
 200                 s1 = md->params[p].regoff;
 201                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 202                         s2 = rd->argintregs[s1];
 203                         if (!md->params[p].inmemory) {           /* register arguments    */
 204                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 205                                         M_INTMOVE(s2, var->regoff);
 206
 207                                 } else {                             /* reg arg -> spilled    */
 208                                     M_LST(s2, REG_SP, var->regoff * 8);
 209                                 }
 210
 211                         } else {                                 /* stack arguments       */
 212                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 213                                         /* + 8 for return address */
 214                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 215
 216                                 } else {                             /* stack arg -> spilled  */
 217                                         var->regoff = parentargs_base + s1 + 1;
 218                                 }
 219                         }
 220
 221                 } else {                                     /* floating args         */
 222                         if (!md->params[p].inmemory) {           /* register arguments    */
 223                                 s2 = rd->argfltregs[s1];
 224                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 225                                         M_FLTMOVE(s2, var->regoff);
 226
 227                                 } else {                                         /* reg arg -> spilled    */
 228                                         M_DST(s2, REG_SP, var->regoff * 8);
 229                                 }
 230
 231                         } else {                                 /* stack arguments       */
 232                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 233                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 234
 235                                 } else {
 236                                         var->regoff = parentargs_base + s1 + 1;
 237                                 }
 238                         }
 239                 }
 240         }  /* end for */
 241
 242         /* save monitorenter argument */
 243
 244 #if defined(USE_THREADS)
 245         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 246                 /* stack offset for monitor argument */
 247
 248                 s1 = rd->memuse;
 249
 250                 if (runverbose) {
 251                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 252
 253                         for (p = 0; p < INT_ARG_CNT; p++)
 254                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 255
 256                         for (p = 0; p < FLT_ARG_CNT; p++)
 257                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 258
 259                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 260                 }
 261
 262                 /* decide which monitor enter function to call */
 263
 264                 if (m->flags & ACC_STATIC) {
 265                         x86_64_mov_imm_reg(cd, (ptrint) m->class, REG_ITMP1);
 266                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, s1 * 8);
 267                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 268                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_staticmonitorenter, REG_ITMP1);
 269                         x86_64_call_reg(cd, REG_ITMP1);
 270
 271                 } else {
 272                         x86_64_test_reg_reg(cd, rd->argintregs[0], rd->argintregs[0]);
 273                         x86_64_jcc(cd, X86_64_CC_Z, 0);
 274                         codegen_addxnullrefs(cd, cd->mcodeptr);
 275                         x86_64_mov_reg_membase(cd, rd->argintregs[0], REG_SP, s1 * 8);
 276                         x86_64_mov_imm_reg(cd, (ptrint) BUILTIN_monitorenter, REG_ITMP1);
 277                         x86_64_call_reg(cd, REG_ITMP1);
 278                 }
 279
 280                 if (runverbose) {
 281                         for (p = 0; p < INT_ARG_CNT; p++)
 282                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 283
 284                         for (p = 0; p < FLT_ARG_CNT; p++)
 285                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 286
 287                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 288                 }
 289         }
 290 #endif
 291
 292         /* Copy argument registers to stack and call trace function with pointer  */
 293         /* to arguments on stack.                                                 */
 294
 295         if (runverbose || opt_stat) {
 296                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 297
 298                 /* save integer argument registers */
 299
 300                 for (p = 0; p < INT_ARG_CNT; p++)
 301                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 302
 303                 /* save float argument registers */
 304
 305                 for (p = 0; p < FLT_ARG_CNT; p++)
 306                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 307
 308                 /* save temporary registers for leaf methods */
 309
 310                 if (m->isleafmethod) {
 311                         for (p = 0; p < INT_TMP_CNT; p++)
 312                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 313
 314                         for (p = 0; p < FLT_TMP_CNT; p++)
 315                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 316                 }
 317
 318                 if (runverbose) {
 319                         /* show integer hex code for float arguments */
 320
 321                         for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 322                                 /* if the paramtype is a float, we have to right shift all    */
 323                                 /* following integer registers                                */
 324
 325                                 if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 326                                         for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 327                                                 M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 328                                         }
 329
 330                                         x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 331                                         l++;
 332                                 }
 333                         }
 334
 335                         x86_64_mov_imm_reg(cd, (ptrint) m, REG_ITMP2);
 336                         x86_64_mov_reg_membase(cd, REG_ITMP2, REG_SP, 0 * 8);
 337                         x86_64_mov_imm_reg(cd, (ptrint) builtin_trace_args, REG_ITMP1);
 338                         x86_64_call_reg(cd, REG_ITMP1);
 339                 }
 340
 341 #if defined(ENABLE_STATISTICS)
 342                 M_MOV_IMM((ptrint) compiledinvokation, REG_ITMP1);
 343                 M_CALL(REG_ITMP1);
 344 #endif
 345
 346                 /* restore integer argument registers */
 347
 348                 for (p = 0; p < INT_ARG_CNT; p++)
 349                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 350
 351                 /* restore float argument registers */
 352
 353                 for (p = 0; p < FLT_ARG_CNT; p++)
 354                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 355
 356                 /* restore temporary registers for leaf methods */
 357
 358                 if (m->isleafmethod) {
 359                         for (p = 0; p < INT_TMP_CNT; p++)
 360                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 361
 362                         for (p = 0; p < FLT_TMP_CNT; p++)
 363                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 364                 }
 365
 366                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 367         }
 368
 369         }
 370
 371         /* end of header generation */
 372
 373         /* walk through all basic blocks */
 374         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 375
 376                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 377
 378                 if (bptr->flags >= BBREACHED) {
 379
 380                         /* branch resolving */
 381
 382                         branchref *bref;
 383                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 384                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 385                                                                   bref->branchpos,
 386                                                                   bptr->mpc);
 387                         }
 388
 389                 /* copy interface registers to their destination */
 390
 391                 src = bptr->instack;
 392                 len = bptr->indepth;
 393                 MCODECHECK(512);
 394
 395 #if defined(ENABLE_LSRA)
 396                 if (opt_lsra) {
 397                         while (src != NULL) {
 398                                 len--;
 399                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 400                                         if (bptr->type == BBTYPE_SBR) {
 401                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 402                                                 if (!(src->flags & INMEMORY))
 403                                                         d= src->regoff;
 404                                                 else
 405                                                         d=REG_ITMP1;
 406                                                 x86_64_pop_reg(cd, d);
 407                                                 store_reg_to_var_int(src, d);
 408
 409                                         } else if (bptr->type == BBTYPE_EXH) {
 410                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 411                                                 if (!(src->flags & INMEMORY))
 412                                                         d= src->regoff;
 413                                                 else
 414                                                         d=REG_ITMP1;
 415                                                 M_INTMOVE(REG_ITMP1, d);
 416                                                 store_reg_to_var_int(src, d);
 417                                         }
 418                                 }
 419                                 src = src->prev;
 420                         }
 421
 422                 } else {
 423 #endif
 424
 425         while (src != NULL) {
 426                         len--;
 427                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 428                                 if (bptr->type == BBTYPE_SBR) {
 429                                         d = reg_of_var(rd, src, REG_ITMP1);
 430                                         x86_64_pop_reg(cd, d);
 431                                         store_reg_to_var_int(src, d);
 432
 433                                 } else if (bptr->type == BBTYPE_EXH) {
 434                                         d = reg_of_var(rd, src, REG_ITMP1);
 435                                         M_INTMOVE(REG_ITMP1, d);
 436                                         store_reg_to_var_int(src, d);
 437                                 }
 438
 439                         } else {
 440                                 d = reg_of_var(rd, src, REG_ITMP1);
 441                                 if ((src->varkind != STACKVAR)) {
 442                                         s2 = src->type;
 443                                         if (IS_FLT_DBL_TYPE(s2)) {
 444                                                 s1 = rd->interfaces[len][s2].regoff;
 445                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 446                                                         M_FLTMOVE(s1, d);
 447
 448                                                 } else {
 449                                                         x86_64_movq_membase_reg(cd, REG_SP, s1 * 8, d);
 450                                                 }
 451                                                 store_reg_to_var_flt(src, d);
 452
 453                                         } else {
 454                                                 s1 = rd->interfaces[len][s2].regoff;
 455                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
 456                                                         M_INTMOVE(s1, d);
 457
 458                                                 } else {
 459                                                         x86_64_mov_membase_reg(cd, REG_SP, s1 * 8, d);
 460                                                 }
 461                                                 store_reg_to_var_int(src, d);
 462                                         }
 463                                 }
 464                         }
 465                         src = src->prev;
 466                 }
 467 #if defined(ENABLE_LSRA)
 468                 }
 469 #endif
 470                 /* walk through all instructions */
 471
 472                 src = bptr->instack;
 473                 len = bptr->icount;
 474                 currentline=0;
 475
 476                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 477                         if (iptr->line != currentline) {
 478                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 479                                 currentline = iptr->line;
 480                         }
 481
 482                         MCODECHECK(1024);                         /* 1KB should be enough */
 483
 484                         switch (iptr->opc) {
 485                         case ICMD_INLINE_START: /* internal ICMDs                         */
 486                         case ICMD_INLINE_END:
 487                                 break;
 488
 489                         case ICMD_NOP:    /* ...  ==> ...                                 */
 490                                 break;
 491
 492                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 493                                 if (src->flags & INMEMORY) {
 494                                         x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
 495
 496                                 } else {
 497                                         x86_64_test_reg_reg(cd, src->regoff, src->regoff);
 498                                 }
 499                                 x86_64_jcc(cd, X86_64_CC_Z, 0);
 500                                 codegen_addxnullrefs(cd, cd->mcodeptr);
 501                                 break;
 502
 503                 /* constant operations ************************************************/
 504
 505                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 506                                       /* op1 = 0, val.i = constant                    */
 507
 508                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 509                         if (iptr->val.i == 0) {
 510                                 M_XOR(d, d);
 511                         } else {
 512                                 M_IMOV_IMM(iptr->val.i, d);
 513                         }
 514                         store_reg_to_var_int(iptr->dst, d);
 515                         break;
 516
 517                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 518                                       /* op1 = 0, val.l = constant                    */
 519
 520                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 521                         if (iptr->val.l == 0) {
 522                                 M_XOR(d, d);
 523                         } else {
 524                                 M_MOV_IMM(iptr->val.l, d);
 525                         }
 526                         store_reg_to_var_int(iptr->dst, d);
 527                         break;
 528
 529                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 530                                       /* op1 = 0, val.f = constant                    */
 531
 532                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 533                         disp = dseg_addfloat(cd, iptr->val.f);
 534                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 535                         store_reg_to_var_flt(iptr->dst, d);
 536                         break;
 537
 538                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 539                                       /* op1 = 0, val.d = constant                    */
 540
 541                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 542                         disp = dseg_adddouble(cd, iptr->val.d);
 543                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 544                         store_reg_to_var_flt(iptr->dst, d);
 545                         break;
 546
 547                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 548                                       /* op1 = 0, val.a = constant                    */
 549
 550                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 551
 552                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 553                                 codegen_addpatchref(cd, cd->mcodeptr,
 554                                                                         PATCHER_aconst,
 555                                                                         (unresolved_class *) iptr->target, 0);
 556
 557                                 if (opt_showdisassemble) {
 558                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 559                                 }
 560
 561                                 M_MOV_IMM((ptrint) iptr->val.a, d);
 562
 563                         } else {
 564                                 if (iptr->val.a == 0) {
 565                                         M_XOR(d, d);
 566                                 } else {
 567                                         M_MOV_IMM((ptrint) iptr->val.a, d);
 568                                 }
 569                         }
 570                         store_reg_to_var_int(iptr->dst, d);
 571                         break;
 572
 573
 574                 /* load/store operations **********************************************/
 575
 576                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 577                                       /* op1 = local variable                         */
 578
 579                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 580                         if ((iptr->dst->varkind == LOCALVAR) &&
 581                             (iptr->dst->varnum == iptr->op1)) {
 582                                 break;
 583                         }
 584                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 585                         if (var->flags & INMEMORY) {
 586                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 587                                 store_reg_to_var_int(iptr->dst, d);
 588
 589                         } else {
 590                                 if (iptr->dst->flags & INMEMORY) {
 591                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 592
 593                                 } else {
 594                                         M_INTMOVE(var->regoff, d);
 595                                 }
 596                         }
 597                         break;
 598
 599                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 600                 case ICMD_ALOAD:      /* op1 = local variable                         */
 601
 602                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 603                         if ((iptr->dst->varkind == LOCALVAR) &&
 604                             (iptr->dst->varnum == iptr->op1)) {
 605                                 break;
 606                         }
 607                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 608                         if (var->flags & INMEMORY) {
 609                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 610                                 store_reg_to_var_int(iptr->dst, d);
 611
 612                         } else {
 613                                 if (iptr->dst->flags & INMEMORY) {
 614                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 615
 616                                 } else {
 617                                         M_INTMOVE(var->regoff, d);
 618                                 }
 619                         }
 620                         break;
 621
 622                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 623                 case ICMD_DLOAD:      /* op1 = local variable                         */
 624
 625                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 626                         if ((iptr->dst->varkind == LOCALVAR) &&
 627                             (iptr->dst->varnum == iptr->op1)) {
 628                                 break;
 629                         }
 630                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 631                         if (var->flags & INMEMORY) {
 632                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 633                                 store_reg_to_var_flt(iptr->dst, d);
 634
 635                         } else {
 636                                 if (iptr->dst->flags & INMEMORY) {
 637                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 638
 639                                 } else {
 640                                         M_FLTMOVE(var->regoff, d);
 641                                 }
 642                         }
 643                         break;
 644
 645                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 646                 case ICMD_LSTORE:     /* op1 = local variable                         */
 647                 case ICMD_ASTORE:
 648
 649                         if ((src->varkind == LOCALVAR) &&
 650                             (src->varnum == iptr->op1)) {
 651                                 break;
 652                         }
 653                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 654                         if (var->flags & INMEMORY) {
 655                                 var_to_reg_int(s1, src, REG_ITMP1);
 656                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 657
 658                         } else {
 659                                 var_to_reg_int(s1, src, var->regoff);
 660                                 M_INTMOVE(s1, var->regoff);
 661                         }
 662                         break;
 663
 664                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 665                 case ICMD_DSTORE:     /* op1 = local variable                         */
 666
 667                         if ((src->varkind == LOCALVAR) &&
 668                             (src->varnum == iptr->op1)) {
 669                                 break;
 670                         }
 671                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 672                         if (var->flags & INMEMORY) {
 673                                 var_to_reg_flt(s1, src, REG_FTMP1);
 674                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 675
 676                         } else {
 677                                 var_to_reg_flt(s1, src, var->regoff);
 678                                 M_FLTMOVE(s1, var->regoff);
 679                         }
 680                         break;
 681
 682
 683                 /* pop/dup/swap operations ********************************************/
 684
 685                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 686
 687                 case ICMD_POP:        /* ..., value  ==> ...                          */
 688                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 689                         break;
 690
 691                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 692                         M_COPY(src, iptr->dst);
 693                         break;
 694
 695                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 696
 697                         M_COPY(src,       iptr->dst);
 698                         M_COPY(src->prev, iptr->dst->prev);
 699                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 700                         break;
 701
 702                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 703
 704                         M_COPY(src,             iptr->dst);
 705                         M_COPY(src->prev,       iptr->dst->prev);
 706                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 707                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 708                         break;
 709
 710                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 711
 712                         M_COPY(src,       iptr->dst);
 713                         M_COPY(src->prev, iptr->dst->prev);
 714                         break;
 715
 716                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 717
 718                         M_COPY(src,             iptr->dst);
 719                         M_COPY(src->prev,       iptr->dst->prev);
 720                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 721                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 722                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 723                         break;
 724
 725                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 726
 727                         M_COPY(src,                   iptr->dst);
 728                         M_COPY(src->prev,             iptr->dst->prev);
 729                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 730                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 731                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 732                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 733                         break;
 734
 735                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 736
 737                         M_COPY(src,       iptr->dst->prev);
 738                         M_COPY(src->prev, iptr->dst);
 739                         break;
 740
 741
 742                 /* integer operations *************************************************/
 743
 744                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 745
 746                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 747                         if (iptr->dst->flags & INMEMORY) {
 748                                 if (src->flags & INMEMORY) {
 749                                         if (src->regoff == iptr->dst->regoff) {
 750                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 751
 752                                         } else {
 753                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 754                                                 x86_64_negl_reg(cd, REG_ITMP1);
 755                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 756                                         }
 757
 758                                 } else {
 759                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 760                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 761                                 }
 762
 763                         } else {
 764                                 if (src->flags & INMEMORY) {
 765                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 766                                         x86_64_negl_reg(cd, d);
 767
 768                                 } else {
 769                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 770                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 771                                 }
 772                         }
 773                         break;
 774
 775                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 776
 777                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 778                         if (iptr->dst->flags & INMEMORY) {
 779                                 if (src->flags & INMEMORY) {
 780                                         if (src->regoff == iptr->dst->regoff) {
 781                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 782
 783                                         } else {
 784                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 785                                                 x86_64_neg_reg(cd, REG_ITMP1);
 786                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 787                                         }
 788
 789                                 } else {
 790                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 791                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 792                                 }
 793
 794                         } else {
 795                                 if (src->flags & INMEMORY) {
 796                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 797                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 798
 799                                 } else {
 800                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 801                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 802                                 }
 803                         }
 804                         break;
 805
 806                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 807
 808                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 809                         if (src->flags & INMEMORY) {
 810                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 811
 812                         } else {
 813                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 814                         }
 815                         store_reg_to_var_int(iptr->dst, d);
 816                         break;
 817
 818                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 819
 820                         var_to_reg_int(s1, src, REG_ITMP1);
 821                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 822                         M_INTMOVE(s1, d);
 823                         store_reg_to_var_int(iptr->dst, d);
 824                         break;
 825
 826                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 827
 828                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 829                         if (src->flags & INMEMORY) {
 830                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 831
 832                         } else {
 833                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 834                         }
 835                         store_reg_to_var_int(iptr->dst, d);
 836                         break;
 837
 838                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 839
 840                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 841                         if (src->flags & INMEMORY) {
 842                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 843
 844                         } else {
 845                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 846                         }
 847                         store_reg_to_var_int(iptr->dst, d);
 848                         break;
 849
 850                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 851
 852                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 853                         if (src->flags & INMEMORY) {
 854                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 855
 856                         } else {
 857                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 858                         }
 859                         store_reg_to_var_int(iptr->dst, d);
 860                         break;
 861
 862
 863                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 864
 865                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 866                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 867                         break;
 868
 869                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 870                                       /* val.i = constant                             */
 871
 872                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 873                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 874                         break;
 875
 876                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 877
 878                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 879                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 880                         break;
 881
 882                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 883                                       /* val.l = constant                             */
 884
 885                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 886                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 887                         break;
 888
 889                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 890
 891                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 892                         if (iptr->dst->flags & INMEMORY) {
 893                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 894                                         if (src->prev->regoff == iptr->dst->regoff) {
 895                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 896                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 897
 898                                         } else {
 899                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 900                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 901                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 902                                         }
 903
 904                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 905                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 906                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 907                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 908
 909                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 910                                         if (src->prev->regoff == iptr->dst->regoff) {
 911                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 912
 913                                         } else {
 914                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 915                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 916                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 917                                         }
 918
 919                                 } else {
 920                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 921                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 922                                 }
 923
 924                         } else {
 925                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 926                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 927                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 928
 929                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 930                                         M_INTMOVE(src->prev->regoff, d);
 931                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 932
 933                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 934                                         /* workaround for reg alloc */
 935                                         if (src->regoff == iptr->dst->regoff) {
 936                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 937                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 938                                                 M_INTMOVE(REG_ITMP1, d);
 939
 940                                         } else {
 941                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 942                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 943                                         }
 944
 945                                 } else {
 946                                         /* workaround for reg alloc */
 947                                         if (src->regoff == iptr->dst->regoff) {
 948                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 949                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 950                                                 M_INTMOVE(REG_ITMP1, d);
 951
 952                                         } else {
 953                                                 M_INTMOVE(src->prev->regoff, d);
 954                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 955                                         }
 956                                 }
 957                         }
 958                         break;
 959
 960                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 961                                       /* val.i = constant                             */
 962
 963                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 964                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
 965                         break;
 966
 967                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 968
 969                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 970                         if (iptr->dst->flags & INMEMORY) {
 971                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 972                                         if (src->prev->regoff == iptr->dst->regoff) {
 973                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 974                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 975
 976                                         } else {
 977                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 978                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 979                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 980                                         }
 981
 982                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 983                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 984                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 985                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 986
 987                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 988                                         if (src->prev->regoff == iptr->dst->regoff) {
 989                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 990
 991                                         } else {
 992                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 993                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 994                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 995                                         }
 996
 997                                 } else {
 998                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 999                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1000                                 }
1001
1002                         } else {
1003                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1004                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1005                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1006
1007                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1008                                         M_INTMOVE(src->prev->regoff, d);
1009                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1010
1011                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1012                                         /* workaround for reg alloc */
1013                                         if (src->regoff == iptr->dst->regoff) {
1014                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1015                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1016                                                 M_INTMOVE(REG_ITMP1, d);
1017
1018                                         } else {
1019                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1020                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1021                                         }
1022
1023                                 } else {
1024                                         /* workaround for reg alloc */
1025                                         if (src->regoff == iptr->dst->regoff) {
1026                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1027                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1028                                                 M_INTMOVE(REG_ITMP1, d);
1029
1030                                         } else {
1031                                                 M_INTMOVE(src->prev->regoff, d);
1032                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1033                                         }
1034                                 }
1035                         }
1036                         break;
1037
1038                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1039                                       /* val.l = constant                             */
1040
1041                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1042                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1043                         break;
1044
1045                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1046
1047                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1048                         if (iptr->dst->flags & INMEMORY) {
1049                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1050                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1051                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1052                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1053
1054                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1055                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1056                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1057                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1058
1059                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1060                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1061                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1062                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1063
1064                                 } else {
1065                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1066                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1067                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1068                                 }
1069
1070                         } else {
1071                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1072                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1073                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1074
1075                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1076                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1077                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1078
1079                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1080                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1081                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1082
1083                                 } else {
1084                                         if (src->regoff == iptr->dst->regoff) {
1085                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1086
1087                                         } else {
1088                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1089                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1090                                         }
1091                                 }
1092                         }
1093                         break;
1094
1095                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1096                                       /* val.i = constant                             */
1097
1098                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1099                         if (iptr->dst->flags & INMEMORY) {
1100                                 if (src->flags & INMEMORY) {
1101                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1102                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1103
1104                                 } else {
1105                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1106                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1107                                 }
1108
1109                         } else {
1110                                 if (src->flags & INMEMORY) {
1111                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1112
1113                                 } else {
1114                                         if (iptr->val.i == 2) {
1115                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1116                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1117
1118                                         } else {
1119                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1120                                         }
1121                                 }
1122                         }
1123                         break;
1124
1125                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1126
1127                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1128                         if (iptr->dst->flags & INMEMORY) {
1129                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1130                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1131                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1132                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1133
1134                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1135                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1136                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1137                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1138
1139                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1140                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1141                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1142                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1143
1144                                 } else {
1145                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1146                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1147                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1148                                 }
1149
1150                         } else {
1151                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1152                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1153                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1154
1155                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1156                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1157                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1158
1159                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1160                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1161                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1162
1163                                 } else {
1164                                         if (src->regoff == iptr->dst->regoff) {
1165                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1166
1167                                         } else {
1168                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1169                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1170                                         }
1171                                 }
1172                         }
1173                         break;
1174
1175                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1176                                       /* val.l = constant                             */
1177
1178                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1179                         if (iptr->dst->flags & INMEMORY) {
1180                                 if (src->flags & INMEMORY) {
1181                                         if (IS_IMM32(iptr->val.l)) {
1182                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1183
1184                                         } else {
1185                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1186                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1187                                         }
1188                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1189
1190                                 } else {
1191                                         if (IS_IMM32(iptr->val.l)) {
1192                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1193
1194                                         } else {
1195                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1196                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1197                                         }
1198                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1199                                 }
1200
1201                         } else {
1202                                 if (src->flags & INMEMORY) {
1203                                         if (IS_IMM32(iptr->val.l)) {
1204                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1205
1206                                         } else {
1207                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1208                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1209                                         }
1210
1211                                 } else {
1212                                         /* should match in many cases */
1213                                         if (iptr->val.l == 2) {
1214                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1215                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1216
1217                                         } else {
1218                                                 if (IS_IMM32(iptr->val.l)) {
1219                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1220
1221                                                 } else {
1222                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1223                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1224                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1225                                                 }
1226                                         }
1227                                 }
1228                         }
1229                         break;
1230
1231                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1232
1233                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1234                 if (src->prev->flags & INMEMORY) {
1235                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1236
1237                         } else {
1238                                 M_INTMOVE(src->prev->regoff, RAX);
1239                         }
1240
1241                         if (src->flags & INMEMORY) {
1242                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1243
1244                         } else {
1245                                 M_INTMOVE(src->regoff, REG_ITMP3);
1246                         }
1247                         gen_div_check(src);
1248
1249                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1250                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1251                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1252                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1253
1254                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1255                         x86_64_cltd(cd);
1256                         x86_64_idivl_reg(cd, REG_ITMP3);
1257
1258                         if (iptr->dst->flags & INMEMORY) {
1259                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1260                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1261
1262                         } else {
1263                                 M_INTMOVE(RAX, iptr->dst->regoff);
1264
1265                                 if (iptr->dst->regoff != RDX) {
1266                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1267                                 }
1268                         }
1269                         break;
1270
1271                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1272                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1273                         if (src->prev->flags & INMEMORY) {
1274                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1275
1276                         } else {
1277                                 M_INTMOVE(src->prev->regoff, RAX);
1278                         }
1279
1280                         if (src->flags & INMEMORY) {
1281                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1282
1283                         } else {
1284                                 M_INTMOVE(src->regoff, REG_ITMP3);
1285                         }
1286                         gen_div_check(src);
1287
1288                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1289
1290                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1291                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1292
1293
1294                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1295                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1296                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1297
1298                         x86_64_cltd(cd);
1299                         x86_64_idivl_reg(cd, REG_ITMP3);
1300
1301                         if (iptr->dst->flags & INMEMORY) {
1302                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1303                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1304
1305                         } else {
1306                                 M_INTMOVE(RDX, iptr->dst->regoff);
1307
1308                                 if (iptr->dst->regoff != RDX) {
1309                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1310                                 }
1311                         }
1312                         break;
1313
1314                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1315                                       /* val.i = constant                             */
1316
1317                         var_to_reg_int(s1, src, REG_ITMP1);
1318                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1319                         M_INTMOVE(s1, REG_ITMP1);
1320                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1321                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1322                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1323                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1324                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1325                         store_reg_to_var_int(iptr->dst, d);
1326                         break;
1327
1328                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1329                                       /* val.i = constant                             */
1330
1331                         var_to_reg_int(s1, src, REG_ITMP1);
1332                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1333                         M_INTMOVE(s1, REG_ITMP1);
1334                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1335                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1336                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1337                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1338                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1339                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1340                         store_reg_to_var_int(iptr->dst, d);
1341                         break;
1342
1343
1344                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1345
1346                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1347
1348                 if (src->prev->flags & INMEMORY) {
1349                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1350
1351                         } else {
1352                                 M_INTMOVE(src->prev->regoff, RAX);
1353                         }
1354
1355                         if (src->flags & INMEMORY) {
1356                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1357
1358                         } else {
1359                                 M_INTMOVE(src->regoff, REG_ITMP3);
1360                         }
1361                         gen_div_check(src);
1362
1363                         /* check as described in jvm spec */
1364                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1365                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1366                         M_BNE(4 + 6);
1367                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1368                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1369
1370                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1371                         x86_64_cqto(cd);
1372                         x86_64_idiv_reg(cd, REG_ITMP3);
1373
1374                         if (iptr->dst->flags & INMEMORY) {
1375                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1376                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1377
1378                         } else {
1379                                 M_INTMOVE(RAX, iptr->dst->regoff);
1380
1381                                 if (iptr->dst->regoff != RDX) {
1382                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1383                                 }
1384                         }
1385                         break;
1386
1387                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1388
1389                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1390                         if (src->prev->flags & INMEMORY) {
1391                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1392
1393                         } else {
1394                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1395                         }
1396
1397                         if (src->flags & INMEMORY) {
1398                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1399
1400                         } else {
1401                                 M_INTMOVE(src->regoff, REG_ITMP3);
1402                         }
1403                         gen_div_check(src);
1404
1405                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1406
1407                         /* check as described in jvm spec */
1408                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1409                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1410                         M_BNE(3 + 4 + 6);
1411
1412 #if 0
1413                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1414 #endif
1415                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1416                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1417                         M_BEQ(2 + 3);                                          /* 6 bytes */
1418
1419                         x86_64_cqto(cd);
1420                         x86_64_idiv_reg(cd, REG_ITMP3);
1421
1422                         if (iptr->dst->flags & INMEMORY) {
1423                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1424                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1425
1426                         } else {
1427                                 M_INTMOVE(RDX, iptr->dst->regoff);
1428
1429                                 if (iptr->dst->regoff != RDX) {
1430                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1431                                 }
1432                         }
1433                         break;
1434
1435                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1436                                       /* val.i = constant                             */
1437
1438                         var_to_reg_int(s1, src, REG_ITMP1);
1439                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1440                         M_INTMOVE(s1, REG_ITMP1);
1441                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1442                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1443                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1444                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1445                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1446                         store_reg_to_var_int(iptr->dst, d);
1447                         break;
1448
1449                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1450                                       /* val.l = constant                             */
1451
1452                         var_to_reg_int(s1, src, REG_ITMP1);
1453                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1454                         M_INTMOVE(s1, REG_ITMP1);
1455                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1456                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1457                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1458                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1459                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1460                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1461                         store_reg_to_var_int(iptr->dst, d);
1462                         break;
1463
1464                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1465
1466                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1467                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1468                         break;
1469
1470                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1471                                       /* val.i = constant                             */
1472
1473                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1474                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1475                         break;
1476
1477                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1478
1479                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1480                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1481                         break;
1482
1483                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1484                                       /* val.i = constant                             */
1485
1486                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1487                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1488                         break;
1489
1490                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1491
1492                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1493                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1494                         break;
1495
1496                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1497                                       /* val.i = constant                             */
1498
1499                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1500                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1501                         break;
1502
1503                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1504
1505                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1506                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1507                         break;
1508
1509         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1510                                           /* val.i = constant                             */
1511
1512                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1513                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1514                         break;
1515
1516                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1517
1518                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1519                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1520                         break;
1521
1522                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1523                                       /* val.i = constant                             */
1524
1525                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1526                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1527                         break;
1528
1529                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1530
1531                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1532                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1533                         break;
1534
1535                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1536                                       /* val.l = constant                             */
1537
1538                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1539                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1540                         break;
1541
1542                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1543
1544                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1545                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1546                         break;
1547
1548                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1549                                       /* val.i = constant                             */
1550
1551                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1552                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1553                         break;
1554
1555                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1556
1557                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1558                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1559                         break;
1560
1561                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1562                                       /* val.l = constant                             */
1563
1564                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1565                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1566                         break;
1567
1568                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1569
1570                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1571                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1572                         break;
1573
1574                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1575                                       /* val.i = constant                             */
1576
1577                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1578                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1579                         break;
1580
1581                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1582
1583                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1584                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1585                         break;
1586
1587                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1588                                       /* val.l = constant                             */
1589
1590                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1591                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1592                         break;
1593
1594                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1595
1596                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1597                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1598                         break;
1599
1600                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1601                                       /* val.i = constant                             */
1602
1603                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1604                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1605                         break;
1606
1607                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1608
1609                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1610                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1611                         break;
1612
1613                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1614                                       /* val.l = constant                             */
1615
1616                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1617                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1618                         break;
1619
1620
1621                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1622                                       /* op1 = variable, val.i = constant             */
1623
1624                         /* using inc and dec is definitely faster than add -- tested      */
1625                         /* with sieve                                                     */
1626
1627                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1628                         d = var->regoff;
1629                         if (var->flags & INMEMORY) {
1630                                 if (iptr->val.i == 1) {
1631                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1632
1633                                 } else if (iptr->val.i == -1) {
1634                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1635
1636                                 } else {
1637                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1638                                 }
1639
1640                         } else {
1641                                 if (iptr->val.i == 1) {
1642                                         x86_64_incl_reg(cd, d);
1643
1644                                 } else if (iptr->val.i == -1) {
1645                                         x86_64_decl_reg(cd, d);
1646
1647                                 } else {
1648                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1649                                 }
1650                         }
1651                         break;
1652
1653
1654                 /* floating operations ************************************************/
1655
1656                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1657
1658                         var_to_reg_flt(s1, src, REG_FTMP1);
1659                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1660                         disp = dseg_adds4(cd, 0x80000000);
1661                         M_FLTMOVE(s1, d);
1662                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1663                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1664                         store_reg_to_var_flt(iptr->dst, d);
1665                         break;
1666
1667                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1668
1669                         var_to_reg_flt(s1, src, REG_FTMP1);
1670                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1671                         disp = dseg_adds8(cd, 0x8000000000000000);
1672                         M_FLTMOVE(s1, d);
1673                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1674                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1675                         store_reg_to_var_flt(iptr->dst, d);
1676                         break;
1677
1678                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1679
1680                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1681                         var_to_reg_flt(s2, src, REG_FTMP2);
1682                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1683                         if (s1 == d) {
1684                                 x86_64_addss_reg_reg(cd, s2, d);
1685                         } else if (s2 == d) {
1686                                 x86_64_addss_reg_reg(cd, s1, d);
1687                         } else {
1688                                 M_FLTMOVE(s1, d);
1689                                 x86_64_addss_reg_reg(cd, s2, d);
1690                         }
1691                         store_reg_to_var_flt(iptr->dst, d);
1692                         break;
1693
1694                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1695
1696                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1697                         var_to_reg_flt(s2, src, REG_FTMP2);
1698                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1699                         if (s1 == d) {
1700                                 x86_64_addsd_reg_reg(cd, s2, d);
1701                         } else if (s2 == d) {
1702                                 x86_64_addsd_reg_reg(cd, s1, d);
1703                         } else {
1704                                 M_FLTMOVE(s1, d);
1705                                 x86_64_addsd_reg_reg(cd, s2, d);
1706                         }
1707                         store_reg_to_var_flt(iptr->dst, d);
1708                         break;
1709
1710                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1711
1712                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1713                         var_to_reg_flt(s2, src, REG_FTMP2);
1714                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1715                         if (s2 == d) {
1716                                 M_FLTMOVE(s2, REG_FTMP2);
1717                                 s2 = REG_FTMP2;
1718                         }
1719                         M_FLTMOVE(s1, d);
1720                         x86_64_subss_reg_reg(cd, s2, d);
1721                         store_reg_to_var_flt(iptr->dst, d);
1722                         break;
1723
1724                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1725
1726                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1727                         var_to_reg_flt(s2, src, REG_FTMP2);
1728                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1729                         if (s2 == d) {
1730                                 M_FLTMOVE(s2, REG_FTMP2);
1731                                 s2 = REG_FTMP2;
1732                         }
1733                         M_FLTMOVE(s1, d);
1734                         x86_64_subsd_reg_reg(cd, s2, d);
1735                         store_reg_to_var_flt(iptr->dst, d);
1736                         break;
1737
1738                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1739
1740                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1741                         var_to_reg_flt(s2, src, REG_FTMP2);
1742                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1743                         if (s1 == d) {
1744                                 x86_64_mulss_reg_reg(cd, s2, d);
1745                         } else if (s2 == d) {
1746                                 x86_64_mulss_reg_reg(cd, s1, d);
1747                         } else {
1748                                 M_FLTMOVE(s1, d);
1749                                 x86_64_mulss_reg_reg(cd, s2, d);
1750                         }
1751                         store_reg_to_var_flt(iptr->dst, d);
1752                         break;
1753
1754                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1755
1756                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1757                         var_to_reg_flt(s2, src, REG_FTMP2);
1758                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1759                         if (s1 == d) {
1760                                 x86_64_mulsd_reg_reg(cd, s2, d);
1761                         } else if (s2 == d) {
1762                                 x86_64_mulsd_reg_reg(cd, s1, d);
1763                         } else {
1764                                 M_FLTMOVE(s1, d);
1765                                 x86_64_mulsd_reg_reg(cd, s2, d);
1766                         }
1767                         store_reg_to_var_flt(iptr->dst, d);
1768                         break;
1769
1770                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1771
1772                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1773                         var_to_reg_flt(s2, src, REG_FTMP2);
1774                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1775                         if (s2 == d) {
1776                                 M_FLTMOVE(s2, REG_FTMP2);
1777                                 s2 = REG_FTMP2;
1778                         }
1779                         M_FLTMOVE(s1, d);
1780                         x86_64_divss_reg_reg(cd, s2, d);
1781                         store_reg_to_var_flt(iptr->dst, d);
1782                         break;
1783
1784                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1785
1786                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1787                         var_to_reg_flt(s2, src, REG_FTMP2);
1788                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1789                         if (s2 == d) {
1790                                 M_FLTMOVE(s2, REG_FTMP2);
1791                                 s2 = REG_FTMP2;
1792                         }
1793                         M_FLTMOVE(s1, d);
1794                         x86_64_divsd_reg_reg(cd, s2, d);
1795                         store_reg_to_var_flt(iptr->dst, d);
1796                         break;
1797
1798                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1799
1800                         var_to_reg_int(s1, src, REG_ITMP1);
1801                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1802                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1803                         store_reg_to_var_flt(iptr->dst, d);
1804                         break;
1805
1806                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1807
1808                         var_to_reg_int(s1, src, REG_ITMP1);
1809                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1810                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1811                         store_reg_to_var_flt(iptr->dst, d);
1812                         break;
1813
1814                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1815
1816                         var_to_reg_int(s1, src, REG_ITMP1);
1817                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1818                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1819                         store_reg_to_var_flt(iptr->dst, d);
1820                         break;
1821
1822                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1823
1824                         var_to_reg_int(s1, src, REG_ITMP1);
1825                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1826                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1827                         store_reg_to_var_flt(iptr->dst, d);
1828                         break;
1829
1830                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1831
1832                         var_to_reg_flt(s1, src, REG_FTMP1);
1833                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1834                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1835                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1836                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1837                         x86_64_jcc(cd, X86_64_CC_NE, a);
1838                         M_FLTMOVE(s1, REG_FTMP1);
1839                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1840                         x86_64_call_reg(cd, REG_ITMP2);
1841                         M_INTMOVE(REG_RESULT, d);
1842                         store_reg_to_var_int(iptr->dst, d);
1843                         break;
1844
1845                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1846
1847                         var_to_reg_flt(s1, src, REG_FTMP1);
1848                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1849                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1850                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1851                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1852                         x86_64_jcc(cd, X86_64_CC_NE, a);
1853                         M_FLTMOVE(s1, REG_FTMP1);
1854                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1855                         x86_64_call_reg(cd, REG_ITMP2);
1856                         M_INTMOVE(REG_RESULT, d);
1857                         store_reg_to_var_int(iptr->dst, d);
1858                         break;
1859
1860                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1861
1862                         var_to_reg_flt(s1, src, REG_FTMP1);
1863                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1864                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1865                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1866                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1867                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1868                         x86_64_jcc(cd, X86_64_CC_NE, a);
1869                         M_FLTMOVE(s1, REG_FTMP1);
1870                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1871                         x86_64_call_reg(cd, REG_ITMP2);
1872                         M_INTMOVE(REG_RESULT, d);
1873                         store_reg_to_var_int(iptr->dst, d);
1874                         break;
1875
1876                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1877
1878                         var_to_reg_flt(s1, src, REG_FTMP1);
1879                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1880                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1881                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1882                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1883                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1884                         x86_64_jcc(cd, X86_64_CC_NE, a);
1885                         M_FLTMOVE(s1, REG_FTMP1);
1886                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1887                         x86_64_call_reg(cd, REG_ITMP2);
1888                         M_INTMOVE(REG_RESULT, d);
1889                         store_reg_to_var_int(iptr->dst, d);
1890                         break;
1891
1892                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1893
1894                         var_to_reg_flt(s1, src, REG_FTMP1);
1895                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1896                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1897                         store_reg_to_var_flt(iptr->dst, d);
1898                         break;
1899
1900                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1901
1902                         var_to_reg_flt(s1, src, REG_FTMP1);
1903                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1904                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1905                         store_reg_to_var_flt(iptr->dst, d);
1906                         break;
1907
1908                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1909                                           /* == => 0, < => 1, > => -1 */
1910
1911                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1912                         var_to_reg_flt(s2, src, REG_FTMP2);
1913                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1914                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1915                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1916                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1917                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1918                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1919                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1920                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1921                         store_reg_to_var_int(iptr->dst, d);
1922                         break;
1923
1924                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1925                                           /* == => 0, < => 1, > => -1 */
1926
1927                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1928                         var_to_reg_flt(s2, src, REG_FTMP2);
1929                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1930                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1931                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1932                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1933                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1934                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1935                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1936                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1937                         store_reg_to_var_int(iptr->dst, d);
1938                         break;
1939
1940                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1941                                           /* == => 0, < => 1, > => -1 */
1942
1943                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1944                         var_to_reg_flt(s2, src, REG_FTMP2);
1945                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1946                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1947                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1948                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1949                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1950                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1951                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1952                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP2, d);    /* treat unordered as GT */
1953                         store_reg_to_var_int(iptr->dst, d);
1954                         break;
1955
1956                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1957                                           /* == => 0, < => 1, > => -1 */
1958
1959                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1960                         var_to_reg_flt(s2, src, REG_FTMP2);
1961                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1962                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
1963                         x86_64_mov_imm_reg(cd, 1, REG_ITMP1);
1964                         x86_64_mov_imm_reg(cd, -1, REG_ITMP2);
1965                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1966                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_B, REG_ITMP1, d);
1967                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_A, REG_ITMP2, d);
1968                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_P, REG_ITMP1, d);    /* treat unordered as LT */
1969                         store_reg_to_var_int(iptr->dst, d);
1970                         break;
1971
1972
1973                 /* memory operations **************************************************/
1974
1975                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
1976
1977                         var_to_reg_int(s1, src, REG_ITMP1);
1978                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1979                         gen_nullptr_check(s1);
1980                         x86_64_movl_membase_reg(cd, s1, OFFSET(java_arrayheader, size), d);
1981                         store_reg_to_var_int(iptr->dst, d);
1982                         break;
1983
1984                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
1985
1986                         var_to_reg_int(s1, src->prev, REG_ITMP1);
1987                         var_to_reg_int(s2, src, REG_ITMP2);
1988                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1989                         if (iptr->op1 == 0) {
1990                                 gen_nullptr_check(s1);
1991                                 gen_bound_check;
1992                         }
1993                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
1994                         store_reg_to_var_int(iptr->dst, d);
1995                         break;
1996
1997                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
1998
1999                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2000                         var_to_reg_int(s2, src, REG_ITMP2);
2001                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2002                         if (iptr->op1 == 0) {
2003                                 gen_nullptr_check(s1);
2004                                 gen_bound_check;
2005                         }
2006                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2007                         store_reg_to_var_int(iptr->dst, d);
2008                         break;
2009
2010                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2011
2012                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2013                         var_to_reg_int(s2, src, REG_ITMP2);
2014                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2015                         if (iptr->op1 == 0) {
2016                                 gen_nullptr_check(s1);
2017                                 gen_bound_check;
2018                         }
2019                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2020                         store_reg_to_var_int(iptr->dst, d);
2021                         break;
2022
2023                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2024
2025                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2026                         var_to_reg_int(s2, src, REG_ITMP2);
2027                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2028                         if (iptr->op1 == 0) {
2029                                 gen_nullptr_check(s1);
2030                                 gen_bound_check;
2031                         }
2032                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2033                         store_reg_to_var_flt(iptr->dst, d);
2034                         break;
2035
2036                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2037
2038                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2039                         var_to_reg_int(s2, src, REG_ITMP2);
2040                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2041                         if (iptr->op1 == 0) {
2042                                 gen_nullptr_check(s1);
2043                                 gen_bound_check;
2044                         }
2045                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2046                         store_reg_to_var_flt(iptr->dst, d);
2047                         break;
2048
2049                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2050
2051                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2052                         var_to_reg_int(s2, src, REG_ITMP2);
2053                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2054                         if (iptr->op1 == 0) {
2055                                 gen_nullptr_check(s1);
2056                                 gen_bound_check;
2057                         }
2058                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2059                         store_reg_to_var_int(iptr->dst, d);
2060                         break;
2061
2062                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2063
2064                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2065                         var_to_reg_int(s2, src, REG_ITMP2);
2066                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2067                         if (iptr->op1 == 0) {
2068                                 gen_nullptr_check(s1);
2069                                 gen_bound_check;
2070                         }
2071                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2072                         store_reg_to_var_int(iptr->dst, d);
2073                         break;
2074
2075                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2076
2077                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2078                         var_to_reg_int(s2, src, REG_ITMP2);
2079                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2080                         if (iptr->op1 == 0) {
2081                                 gen_nullptr_check(s1);
2082                                 gen_bound_check;
2083                         }
2084                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2085                         store_reg_to_var_int(iptr->dst, d);
2086                         break;
2087
2088
2089                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2090
2091                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2092                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2093                         if (iptr->op1 == 0) {
2094                                 gen_nullptr_check(s1);
2095                                 gen_bound_check;
2096                         }
2097                         var_to_reg_int(s3, src, REG_ITMP3);
2098                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2099                         break;
2100
2101                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2102
2103                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2104                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2105                         if (iptr->op1 == 0) {
2106                                 gen_nullptr_check(s1);
2107                                 gen_bound_check;
2108                         }
2109                         var_to_reg_int(s3, src, REG_ITMP3);
2110                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2111                         break;
2112
2113                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2114
2115                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2116                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2117                         if (iptr->op1 == 0) {
2118                                 gen_nullptr_check(s1);
2119                                 gen_bound_check;
2120                         }
2121                         var_to_reg_flt(s3, src, REG_FTMP3);
2122                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2123                         break;
2124
2125                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2126
2127                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2128                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2129                         if (iptr->op1 == 0) {
2130                                 gen_nullptr_check(s1);
2131                                 gen_bound_check;
2132                         }
2133                         var_to_reg_flt(s3, src, REG_FTMP3);
2134                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2135                         break;
2136
2137                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2138
2139                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2140                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2141                         if (iptr->op1 == 0) {
2142                                 gen_nullptr_check(s1);
2143                                 gen_bound_check;
2144                         }
2145                         var_to_reg_int(s3, src, REG_ITMP3);
2146                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2147                         break;
2148
2149                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2150
2151                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2152                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2153                         if (iptr->op1 == 0) {
2154                                 gen_nullptr_check(s1);
2155                                 gen_bound_check;
2156                         }
2157                         var_to_reg_int(s3, src, REG_ITMP3);
2158                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2159                         break;
2160
2161                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2162
2163                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2164                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2165                         if (iptr->op1 == 0) {
2166                                 gen_nullptr_check(s1);
2167                                 gen_bound_check;
2168                         }
2169                         var_to_reg_int(s3, src, REG_ITMP3);
2170                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2171                         break;
2172
2173                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2174
2175                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2176                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2177                         if (iptr->op1 == 0) {
2178                                 gen_nullptr_check(s1);
2179                                 gen_bound_check;
2180                         }
2181                         var_to_reg_int(s3, src, REG_ITMP3);
2182
2183                         M_MOV(s1, rd->argintregs[0]);
2184                         M_MOV(s3, rd->argintregs[1]);
2185                         M_MOV_IMM((ptrint) BUILTIN_canstore, REG_ITMP1);
2186                         M_CALL(REG_ITMP1);
2187                         M_TEST(REG_RESULT);
2188                         M_BEQ(0);
2189                         codegen_addxstorerefs(cd, cd->mcodeptr);
2190
2191                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2192                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2193                         var_to_reg_int(s3, src, REG_ITMP3);
2194                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2195                         break;
2196
2197
2198                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2199
2200                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2201                         var_to_reg_int(s2, src, REG_ITMP2);
2202                         if (iptr->op1 == 0) {
2203                                 gen_nullptr_check(s1);
2204                                 gen_bound_check;
2205                         }
2206                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2207                         break;
2208
2209                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2210
2211                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2212                         var_to_reg_int(s2, src, REG_ITMP2);
2213                         if (iptr->op1 == 0) {
2214                                 gen_nullptr_check(s1);
2215                                 gen_bound_check;
2216                         }
2217                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2218                         break;
2219
2220                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2221
2222                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2223                         var_to_reg_int(s2, src, REG_ITMP2);
2224                         if (iptr->op1 == 0) {
2225                                 gen_nullptr_check(s1);
2226                                 gen_bound_check;
2227                         }
2228                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2229                         break;
2230
2231                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2232
2233                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2234                         var_to_reg_int(s2, src, REG_ITMP2);
2235                         if (iptr->op1 == 0) {
2236                                 gen_nullptr_check(s1);
2237                                 gen_bound_check;
2238                         }
2239                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2240                         break;
2241
2242                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2243
2244                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2245                         var_to_reg_int(s2, src, REG_ITMP2);
2246                         if (iptr->op1 == 0) {
2247                                 gen_nullptr_check(s1);
2248                                 gen_bound_check;
2249                         }
2250
2251                         if (IS_IMM32(iptr->val.l)) {
2252                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2253
2254                         } else {
2255                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2256                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2257                         }
2258                         break;
2259
2260                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2261
2262                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2263                         var_to_reg_int(s2, src, REG_ITMP2);
2264                         if (iptr->op1 == 0) {
2265                                 gen_nullptr_check(s1);
2266                                 gen_bound_check;
2267                         }
2268                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2269                         break;
2270
2271
2272                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2273                                       /* op1 = type, val.a = field address            */
2274
2275                         if (!iptr->val.a) {
2276                                 disp = dseg_addaddress(cd, NULL);
2277
2278                                 codegen_addpatchref(cd, cd->mcodeptr,
2279                                                                         PATCHER_get_putstatic,
2280                                                                         (unresolved_field *) iptr->target, disp);
2281
2282                                 if (opt_showdisassemble) {
2283                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2284                                 }
2285
2286                         } else {
2287                                 fieldinfo *fi = iptr->val.a;
2288
2289                                 disp = dseg_addaddress(cd, &(fi->value));
2290
2291                                 if (!(fi->class->state & CLASS_INITIALIZED)) {
2292                                         codegen_addpatchref(cd, cd->mcodeptr,
2293                                                                                 PATCHER_clinit, fi->class, 0);
2294
2295                                         if (opt_showdisassemble) {
2296                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2297                                         }
2298                                 }
2299                         }
2300
2301                         /* This approach is much faster than moving the field address     */
2302                         /* inline into a register. */
2303                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2304                         switch (iptr->op1) {
2305                         case TYPE_INT:
2306                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2307                                 x86_64_movl_membase_reg(cd, REG_ITMP2, 0, d);
2308                                 store_reg_to_var_int(iptr->dst, d);
2309                                 break;
2310                         case TYPE_LNG:
2311                         case TYPE_ADR:
2312                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2313                                 x86_64_mov_membase_reg(cd, REG_ITMP2, 0, d);
2314                                 store_reg_to_var_int(iptr->dst, d);
2315                                 break;
2316                         case TYPE_FLT:
2317                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2318                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2319                                 store_reg_to_var_flt(iptr->dst, d);
2320                                 break;
2321                         case TYPE_DBL:
2322                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2323                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2324                                 store_reg_to_var_flt(iptr->dst, d);
2325                                 break;
2326                         }
2327                         break;
2328
2329                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2330                                       /* op1 = type, val.a = field address            */
2331
2332                         if (!iptr->val.a) {
2333                                 disp = dseg_addaddress(cd, NULL);
2334
2335                                 codegen_addpatchref(cd, cd->mcodeptr,
2336                                                                         PATCHER_get_putstatic,
2337                                                                         (unresolved_field *) iptr->target, disp);
2338
2339                                 if (opt_showdisassemble) {
2340                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2341                                 }
2342
2343                         } else {
2344                                 fieldinfo *fi = iptr->val.a;
2345
2346                                 disp = dseg_addaddress(cd, &(fi->value));
2347
2348                                 if (!(fi->class->state & CLASS_INITIALIZED)) {
2349                                         codegen_addpatchref(cd, cd->mcodeptr,
2350                                                                                 PATCHER_clinit, fi->class, 0);
2351
2352                                         if (opt_showdisassemble) {
2353                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2354                                         }
2355                                 }
2356                         }
2357
2358                         /* This approach is much faster than moving the field address     */
2359                         /* inline into a register. */
2360                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP2);
2361                         switch (iptr->op1) {
2362                         case TYPE_INT:
2363                                 var_to_reg_int(s2, src, REG_ITMP1);
2364                                 x86_64_movl_reg_membase(cd, s2, REG_ITMP2, 0);
2365                                 break;
2366                         case TYPE_LNG:
2367                         case TYPE_ADR:
2368                                 var_to_reg_int(s2, src, REG_ITMP1);
2369                                 x86_64_mov_reg_membase(cd, s2, REG_ITMP2, 0);
2370                                 break;
2371                         case TYPE_FLT:
2372                                 var_to_reg_flt(s2, src, REG_FTMP1);
2373                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2374                                 break;
2375                         case TYPE_DBL:
2376                                 var_to_reg_flt(s2, src, REG_FTMP1);
2377                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2378                                 break;
2379                         }
2380                         break;
2381
2382                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2383                                           /* val = value (in current instruction)     */
2384                                           /* op1 = type, val.a = field address (in    */
2385                                           /* following NOP)                           */
2386
2387                         if (!iptr[1].val.a) {
2388                                 disp = dseg_addaddress(cd, NULL);
2389
2390                                 codegen_addpatchref(cd, cd->mcodeptr,
2391                                                                         PATCHER_get_putstatic,
2392                                                                         (unresolved_field *) iptr[1].target, disp);
2393
2394                                 if (opt_showdisassemble) {
2395                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2396                                 }
2397
2398                         } else {
2399                                 fieldinfo *fi = iptr[1].val.a;
2400
2401                                 disp = dseg_addaddress(cd, &(fi->value));
2402
2403                                 if (!(fi->class->state & CLASS_INITIALIZED)) {
2404                                         codegen_addpatchref(cd, cd->mcodeptr,
2405                                                                                 PATCHER_clinit, fi->class, 0);
2406
2407                                         if (opt_showdisassemble) {
2408                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2409                                         }
2410                                 }
2411                         }
2412
2413                         /* This approach is much faster than moving the field address     */
2414                         /* inline into a register. */
2415                         x86_64_mov_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
2416                         switch (iptr->op1) {
2417                         case TYPE_INT:
2418                         case TYPE_FLT:
2419                                 x86_64_movl_imm_membase(cd, iptr->val.i, REG_ITMP1, 0);
2420                                 break;
2421                         case TYPE_LNG:
2422                         case TYPE_ADR:
2423                         case TYPE_DBL:
2424                                 if (IS_IMM32(iptr->val.l)) {
2425                                         x86_64_mov_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2426                                 } else {
2427                                         x86_64_movl_imm_membase(cd, iptr->val.l, REG_ITMP1, 0);
2428                                         x86_64_movl_imm_membase(cd, iptr->val.l >> 32, REG_ITMP1, 4);
2429                                 }
2430                                 break;
2431                         }
2432                         break;
2433
2434                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2435                                       /* op1 = type, val.i = field offset             */
2436
2437                         var_to_reg_int(s1, src, REG_ITMP1);
2438                         gen_nullptr_check(s1);
2439
2440                         if (!iptr->val.a) {
2441                                 codegen_addpatchref(cd, cd->mcodeptr,
2442                                                                         PATCHER_get_putfield,
2443                                                                         (unresolved_field *) iptr->target, 0);
2444
2445                                 if (opt_showdisassemble) {
2446                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2447                                 }
2448
2449                                 a = 0;
2450
2451                         } else {
2452                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2453                         }
2454
2455                         switch (iptr->op1) {
2456                         case TYPE_INT:
2457                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2458                                 x86_64_movl_membase32_reg(cd, s1, a, d);
2459                                 store_reg_to_var_int(iptr->dst, d);
2460                                 break;
2461                         case TYPE_LNG:
2462                         case TYPE_ADR:
2463                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2464                                 x86_64_mov_membase32_reg(cd, s1, a, d);
2465                                 store_reg_to_var_int(iptr->dst, d);
2466                                 break;
2467                         case TYPE_FLT:
2468                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2469                                 x86_64_movss_membase32_reg(cd, s1, a, d);
2470                                 store_reg_to_var_flt(iptr->dst, d);
2471                                 break;
2472                         case TYPE_DBL:
2473                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2474                                 x86_64_movsd_membase32_reg(cd, s1, a, d);
2475                                 store_reg_to_var_flt(iptr->dst, d);
2476                                 break;
2477                         }
2478                         break;
2479
2480                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2481                                       /* op1 = type, val.i = field offset             */
2482
2483                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2484                         gen_nullptr_check(s1);
2485                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2486                                 var_to_reg_int(s2, src, REG_ITMP2);
2487                         } else {
2488                                 var_to_reg_flt(s2, src, REG_FTMP2);
2489                         }
2490
2491                         if (!iptr->val.a) {
2492                                 codegen_addpatchref(cd, cd->mcodeptr,
2493                                                                         PATCHER_get_putfield,
2494                                                                         (unresolved_field *) iptr->target, 0);
2495
2496                                 if (opt_showdisassemble) {
2497                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2498                                 }
2499
2500                                 a = 0;
2501
2502                         } else {
2503                                 a = ((fieldinfo *) (iptr->val.a))->offset;
2504                         }
2505
2506                         switch (iptr->op1) {
2507                         case TYPE_INT:
2508                                 x86_64_movl_reg_membase32(cd, s2, s1, a);
2509                                 break;
2510                         case TYPE_LNG:
2511                         case TYPE_ADR:
2512                                 x86_64_mov_reg_membase32(cd, s2, s1, a);
2513                                 break;
2514                         case TYPE_FLT:
2515                                 x86_64_movss_reg_membase32(cd, s2, s1, a);
2516                                 break;
2517                         case TYPE_DBL:
2518                                 x86_64_movsd_reg_membase32(cd, s2, s1, a);
2519                                 break;
2520                         }
2521                         break;
2522
2523                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2524                                           /* val = value (in current instruction)     */
2525                                           /* op1 = type, val.a = field address (in    */
2526                                           /* following NOP)                           */
2527
2528                         var_to_reg_int(s1, src, REG_ITMP1);
2529                         gen_nullptr_check(s1);
2530
2531                         if (!iptr[1].val.a) {
2532                                 codegen_addpatchref(cd, cd->mcodeptr,
2533                                                                         PATCHER_putfieldconst,
2534                                                                         (unresolved_field *) iptr[1].target, 0);
2535
2536                                 if (opt_showdisassemble) {
2537                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2538                                 }
2539
2540                                 a = 0;
2541
2542                         } else {
2543                                 a = ((fieldinfo *) (iptr[1].val.a))->offset;
2544                         }
2545
2546                         switch (iptr->op1) {
2547                         case TYPE_INT:
2548                         case TYPE_FLT:
2549                                 x86_64_movl_imm_membase32(cd, iptr->val.i, s1, a);
2550                                 break;
2551                         case TYPE_LNG:
2552                         case TYPE_ADR:
2553                         case TYPE_DBL:
2554                                 /* We can only optimize the move, if the class is resolved.   */
2555                                 /* Otherwise we don't know what to patch.                     */
2556                                 if (iptr[1].val.a && IS_IMM32(iptr->val.l)) {
2557                                         x86_64_mov_imm_membase32(cd, iptr->val.l, s1, a);
2558                                 } else {
2559                                         x86_64_movl_imm_membase32(cd, iptr->val.l, s1, a);
2560                                         x86_64_movl_imm_membase32(cd, iptr->val.l >> 32, s1, a + 4);
2561                                 }
2562                                 break;
2563                         }
2564                         break;
2565
2566
2567                 /* branch operations **************************************************/
2568
2569                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2570
2571                         var_to_reg_int(s1, src, REG_ITMP1);
2572                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2573
2574 #ifdef ENABLE_VERIFIER
2575                         if (iptr->val.a) {
2576                                 codegen_addpatchref(cd, cd->mcodeptr,
2577                                                                         PATCHER_athrow_areturn,
2578                                                                         (unresolved_class *) iptr->val.a, 0);
2579
2580                                 if (opt_showdisassemble) {
2581                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2582                                 }
2583                         }
2584 #endif /* ENABLE_VERIFIER */
2585
2586                         M_CALL_IMM(0);                            /* passing exception pc */
2587                         M_POP(REG_ITMP2_XPC);
2588
2589                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
2590                         M_JMP(REG_ITMP3);
2591                         break;
2592
2593                 case ICMD_GOTO:         /* ... ==> ...                                */
2594                                         /* op1 = target JavaVM pc                     */
2595
2596                         M_JMP_IMM(0);
2597                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2598                         break;
2599
2600                 case ICMD_JSR:          /* ... ==> ...                                */
2601                                         /* op1 = target JavaVM pc                     */
2602
2603                         M_CALL_IMM(0);
2604                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2605                         break;
2606
2607                 case ICMD_RET:          /* ... ==> ...                                */
2608                                         /* op1 = local variable                       */
2609
2610                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2611                         var_to_reg_int(s1, var, REG_ITMP1);
2612                         M_JMP(s1);
2613                         break;
2614
2615                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2616                                         /* op1 = target JavaVM pc                     */
2617
2618                         if (src->flags & INMEMORY) {
2619                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2620
2621                         } else {
2622                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2623                         }
2624                         x86_64_jcc(cd, X86_64_CC_E, 0);
2625                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2626                         break;
2627
2628                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2629                                         /* op1 = target JavaVM pc                     */
2630
2631                         if (src->flags & INMEMORY) {
2632                                 x86_64_alu_imm_membase(cd, X86_64_CMP, 0, REG_SP, src->regoff * 8);
2633
2634                         } else {
2635                                 x86_64_test_reg_reg(cd, src->regoff, src->regoff);
2636                         }
2637                         x86_64_jcc(cd, X86_64_CC_NE, 0);
2638                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2639                         break;
2640
2641                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2642                                         /* op1 = target JavaVM pc, val.i = constant   */
2643
2644                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2645                         break;
2646
2647                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2648                                         /* op1 = target JavaVM pc, val.i = constant   */
2649
2650                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2651                         break;
2652
2653                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2654                                         /* op1 = target JavaVM pc, val.i = constant   */
2655
2656                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2657                         break;
2658
2659                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2660                                         /* op1 = target JavaVM pc, val.i = constant   */
2661
2662                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2663                         break;
2664
2665                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2666                                         /* op1 = target JavaVM pc, val.i = constant   */
2667
2668                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2669                         break;
2670
2671                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2672                                         /* op1 = target JavaVM pc, val.i = constant   */
2673
2674                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2675                         break;
2676
2677                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2678                                         /* op1 = target JavaVM pc, val.l = constant   */
2679
2680                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2681                         break;
2682
2683                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2684                                         /* op1 = target JavaVM pc, val.l = constant   */
2685
2686                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2687                         break;
2688
2689                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2690                                         /* op1 = target JavaVM pc, val.l = constant   */
2691
2692                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2693                         break;
2694
2695                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2696                                         /* op1 = target JavaVM pc, val.l = constant   */
2697
2698                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2699                         break;
2700
2701                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2702                                         /* op1 = target JavaVM pc, val.l = constant   */
2703
2704                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2705                         break;
2706
2707                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2708                                         /* op1 = target JavaVM pc, val.l = constant   */
2709
2710                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2711                         break;
2712
2713                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2714                                         /* op1 = target JavaVM pc                     */
2715
2716                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2717                         break;
2718
2719                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2720                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2721
2722                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2723                         break;
2724
2725                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2726                                         /* op1 = target JavaVM pc                     */
2727
2728                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2729                         break;
2730
2731                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2732                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2733
2734                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2735                         break;
2736
2737                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2738                                         /* op1 = target JavaVM pc                     */
2739
2740                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2741                         break;
2742
2743                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2744                                     /* op1 = target JavaVM pc                     */
2745
2746                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2747                         break;
2748
2749                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2750                                         /* op1 = target JavaVM pc                     */
2751
2752                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2753                         break;
2754
2755                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2756                                 /* op1 = target JavaVM pc                     */
2757
2758                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2759                         break;
2760
2761                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2762                                         /* op1 = target JavaVM pc                     */
2763
2764                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2765                         break;
2766
2767                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2768                                         /* op1 = target JavaVM pc                     */
2769
2770                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2771                         break;
2772
2773                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2774                                         /* op1 = target JavaVM pc                     */
2775
2776                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2777                         break;
2778
2779                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2780                                     /* op1 = target JavaVM pc                     */
2781
2782                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2783                         break;
2784
2785                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2786
2787                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2788                         break;
2789
2790                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2791                                         /* val.i = constant                           */
2792
2793                         var_to_reg_int(s1, src, REG_ITMP1);
2794                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2795                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2796                                 if (s1 == d) {
2797                                         M_INTMOVE(s1, REG_ITMP1);
2798                                         s1 = REG_ITMP1;
2799                                 }
2800                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2801                         }
2802                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2803                         x86_64_testl_reg_reg(cd, s1, s1);
2804                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_E, REG_ITMP2, d);
2805                         store_reg_to_var_int(iptr->dst, d);
2806                         break;
2807
2808                 case ICMD_IFNE_ICONST:  /* ..., value ==> ..., constant               */
2809                                         /* val.i = constant                           */
2810
2811                         var_to_reg_int(s1, src, REG_ITMP1);
2812                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2813                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2814                                 if (s1 == d) {
2815                                         M_INTMOVE(s1, REG_ITMP1);
2816                                         s1 = REG_ITMP1;
2817                                 }
2818                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2819                         }
2820                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2821                         x86_64_testl_reg_reg(cd, s1, s1);
2822                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_NE, REG_ITMP2, d);
2823                         store_reg_to_var_int(iptr->dst, d);
2824                         break;
2825
2826                 case ICMD_IFLT_ICONST:  /* ..., value ==> ..., constant               */
2827                                         /* val.i = constant                           */
2828
2829                         var_to_reg_int(s1, src, REG_ITMP1);
2830                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2831                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2832                                 if (s1 == d) {
2833                                         M_INTMOVE(s1, REG_ITMP1);
2834                                         s1 = REG_ITMP1;
2835                                 }
2836                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2837                         }
2838                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2839                         x86_64_testl_reg_reg(cd, s1, s1);
2840                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_L, REG_ITMP2, d);
2841                         store_reg_to_var_int(iptr->dst, d);
2842                         break;
2843
2844                 case ICMD_IFGE_ICONST:  /* ..., value ==> ..., constant               */
2845                                         /* val.i = constant                           */
2846
2847                         var_to_reg_int(s1, src, REG_ITMP1);
2848                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2849                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2850                                 if (s1 == d) {
2851                                         M_INTMOVE(s1, REG_ITMP1);
2852                                         s1 = REG_ITMP1;
2853                                 }
2854                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2855                         }
2856                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2857                         x86_64_testl_reg_reg(cd, s1, s1);
2858                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_GE, REG_ITMP2, d);
2859                         store_reg_to_var_int(iptr->dst, d);
2860                         break;
2861
2862                 case ICMD_IFGT_ICONST:  /* ..., value ==> ..., constant               */
2863                                         /* val.i = constant                           */
2864
2865                         var_to_reg_int(s1, src, REG_ITMP1);
2866                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2867                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2868                                 if (s1 == d) {
2869                                         M_INTMOVE(s1, REG_ITMP1);
2870                                         s1 = REG_ITMP1;
2871                                 }
2872                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2873                         }
2874                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2875                         x86_64_testl_reg_reg(cd, s1, s1);
2876                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP2, d);
2877                         store_reg_to_var_int(iptr->dst, d);
2878                         break;
2879
2880                 case ICMD_IFLE_ICONST:  /* ..., value ==> ..., constant               */
2881                                         /* val.i = constant                           */
2882
2883                         var_to_reg_int(s1, src, REG_ITMP1);
2884                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2885                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2886                                 if (s1 == d) {
2887                                         M_INTMOVE(s1, REG_ITMP1);
2888                                         s1 = REG_ITMP1;
2889                                 }
2890                                 x86_64_movl_imm_reg(cd, iptr[1].val.i, d);
2891                         }
2892                         x86_64_movl_imm_reg(cd, iptr->val.i, REG_ITMP2);
2893                         x86_64_testl_reg_reg(cd, s1, s1);
2894                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, d);
2895                         store_reg_to_var_int(iptr->dst, d);
2896                         break;
2897
2898
2899                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2900                 case ICMD_LRETURN:
2901
2902                         var_to_reg_int(s1, src, REG_RESULT);
2903                         M_INTMOVE(s1, REG_RESULT);
2904                         goto nowperformreturn;
2905
2906                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2907
2908                         var_to_reg_int(s1, src, REG_RESULT);
2909                         M_INTMOVE(s1, REG_RESULT);
2910
2911 #ifdef ENABLE_VERIFIER
2912                         if (iptr->val.a) {
2913                                 codegen_addpatchref(cd, cd->mcodeptr,
2914                                                                         PATCHER_athrow_areturn,
2915                                                                         (unresolved_class *) iptr->val.a, 0);
2916
2917                                 if (opt_showdisassemble) {
2918                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2919                                 }
2920                         }
2921 #endif /* ENABLE_VERIFIER */
2922                         goto nowperformreturn;
2923
2924                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2925                 case ICMD_DRETURN:
2926
2927                         var_to_reg_flt(s1, src, REG_FRESULT);
2928                         M_FLTMOVE(s1, REG_FRESULT);
2929                         goto nowperformreturn;
2930
2931                 case ICMD_RETURN:      /* ...  ==> ...                                */
2932
2933 nowperformreturn:
2934                         {
2935                         s4 i, p;
2936
2937                         p = parentargs_base;
2938
2939                         /* call trace function */
2940                         if (runverbose) {
2941                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2942
2943                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2944                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2945
2946                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2947                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2948                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2949                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2950
2951                                 x86_64_mov_imm_reg(cd, (u8) builtin_displaymethodstop, REG_ITMP1);
2952                                 x86_64_call_reg(cd, REG_ITMP1);
2953
2954                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
2955                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
2956
2957                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
2958                         }
2959
2960 #if defined(USE_THREADS)
2961                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
2962                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
2963
2964                                 /* we need to save the proper return value */
2965                                 switch (iptr->opc) {
2966                                 case ICMD_IRETURN:
2967                                 case ICMD_ARETURN:
2968                                 case ICMD_LRETURN:
2969                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
2970                                         break;
2971                                 case ICMD_FRETURN:
2972                                 case ICMD_DRETURN:
2973                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
2974                                         break;
2975                                 }
2976
2977                                 M_MOV_IMM((ptrint) builtin_monitorexit, REG_ITMP1);
2978                                 M_CALL(REG_ITMP1);
2979
2980                                 /* and now restore the proper return value */
2981                                 switch (iptr->opc) {
2982                                 case ICMD_IRETURN:
2983                                 case ICMD_ARETURN:
2984                                 case ICMD_LRETURN:
2985                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
2986                                         break;
2987                                 case ICMD_FRETURN:
2988                                 case ICMD_DRETURN:
2989                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
2990                                         break;
2991                                 }
2992                         }
2993 #endif
2994
2995                         /* restore saved registers */
2996
2997                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
2998                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
2999                         }
3000                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3001                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3002                         }
3003
3004                         /* deallocate stack */
3005
3006                         if (parentargs_base)
3007                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
3008
3009                         M_RET;
3010                         }
3011                         break;
3012
3013
3014                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3015                         {
3016                                 s4 i, l, *s4ptr;
3017                                 void **tptr;
3018
3019                                 tptr = (void **) iptr->target;
3020
3021                                 s4ptr = iptr->val.a;
3022                                 l = s4ptr[1];                          /* low     */
3023                                 i = s4ptr[2];                          /* high    */
3024
3025                                 var_to_reg_int(s1, src, REG_ITMP1);
3026                                 M_INTMOVE(s1, REG_ITMP1);
3027                                 if (l != 0) {
3028                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3029                                 }
3030                                 i = i - l + 1;
3031
3032                 /* range check */
3033                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3034                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3035
3036                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3037                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3038
3039                                 /* build jump table top down and use address of lowest entry */
3040
3041                 /* s4ptr += 3 + i; */
3042                                 tptr += i;
3043
3044                                 while (--i >= 0) {
3045                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3046                                         --tptr;
3047                                 }
3048
3049                                 /* length of dataseg after last dseg_addtarget is used by load */
3050
3051                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3052                                 dseg_adddata(cd, cd->mcodeptr);
3053                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3054                                 x86_64_jmp_reg(cd, REG_ITMP1);
3055                         }
3056                         break;
3057
3058
3059                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3060                         {
3061                                 s4 i, l, val, *s4ptr;
3062                                 void **tptr;
3063
3064                                 tptr = (void **) iptr->target;
3065
3066                                 s4ptr = iptr->val.a;
3067                                 l = s4ptr[0];                          /* default  */
3068                                 i = s4ptr[1];                          /* count    */
3069
3070                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3071                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3072                                 while (--i >= 0) {
3073                                         s4ptr += 2;
3074                                         ++tptr;
3075
3076                                         val = s4ptr[0];
3077                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3078                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3079                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3080                                 }
3081
3082                                 x86_64_jmp_imm(cd, 0);
3083
3084                                 tptr = (void **) iptr->target;
3085                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3086                         }
3087                         break;
3088
3089
3090                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3091                                         /* op1 = arg count val.a = builtintable entry */
3092
3093                         bte = iptr->val.a;
3094                         md = bte->md;
3095                         goto gen_method;
3096
3097                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3098                                         /* op1 = arg count, val.a = method pointer    */
3099
3100                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3101                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3102                 case ICMD_INVOKEINTERFACE:
3103
3104                         lm = iptr->val.a;
3105
3106                         if (lm == NULL) {
3107                                 unresolved_method *um = iptr->target;
3108                                 md = um->methodref->parseddesc.md;
3109                         } else {
3110                                 md = lm->parseddesc;
3111                         }
3112
3113 gen_method:
3114                         s3 = md->paramcount;
3115
3116                         MCODECHECK((20 * s3) + 128);
3117
3118                         /* copy arguments to registers or stack location */
3119
3120                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3121                                 if (src->varkind == ARGVAR)
3122                                         continue;
3123                                 if (IS_INT_LNG_TYPE(src->type)) {
3124                                         if (!md->params[s3].inmemory) {
3125                                                 s1 = rd->argintregs[md->params[s3].regoff];
3126                                                 var_to_reg_int(d, src, s1);
3127                                                 M_INTMOVE(d, s1);
3128                                         } else {
3129                                                 var_to_reg_int(d, src, REG_ITMP1);
3130                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3131                                         }
3132
3133                                 } else {
3134                                         if (!md->params[s3].inmemory) {
3135                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3136                                                 var_to_reg_flt(d, src, s1);
3137                                                 M_FLTMOVE(d, s1);
3138                                         } else {
3139                                                 var_to_reg_flt(d, src, REG_FTMP1);
3140                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3141                                         }
3142                                 }
3143                         }
3144
3145                         switch (iptr->opc) {
3146                         case ICMD_BUILTIN:
3147                                 a = (ptrint) bte->fp;
3148                                 d = md->returntype.type;
3149
3150                                 M_MOV_IMM(a, REG_ITMP1);
3151                                 M_CALL(REG_ITMP1);
3152
3153                                 /* if op1 == true, we need to check for an exception */
3154
3155                                 if (iptr->op1 == true) {
3156                                         M_TEST(REG_RESULT);
3157                                         M_BEQ(0);
3158                                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3159                                 }
3160                                 break;
3161
3162                         case ICMD_INVOKESPECIAL:
3163                                 M_TEST(rd->argintregs[0]);
3164                                 M_BEQ(0);
3165                                 codegen_addxnullrefs(cd, cd->mcodeptr);
3166
3167                                 /* first argument contains pointer */
3168 /*                              gen_nullptr_check(rd->argintregs[0]); */
3169
3170                                 /* access memory for hardware nullptr */
3171 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3172
3173                                 /* fall through */
3174
3175                         case ICMD_INVOKESTATIC:
3176                                 if (lm == NULL) {
3177                                         unresolved_method *um = iptr->target;
3178
3179                                         codegen_addpatchref(cd, cd->mcodeptr,
3180                                                                                 PATCHER_invokestatic_special, um, 0);
3181
3182                                         if (opt_showdisassemble) {
3183                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3184                                         }
3185
3186                                         a = 0;
3187                                         d = um->methodref->parseddesc.md->returntype.type;
3188
3189                                 } else {
3190                                         a = (ptrint) lm->stubroutine;
3191                                         d = lm->parseddesc->returntype.type;
3192                                 }
3193
3194                                 M_MOV_IMM(a, REG_ITMP2);
3195                                 M_CALL(REG_ITMP2);
3196                                 break;
3197
3198                         case ICMD_INVOKEVIRTUAL:
3199                                 gen_nullptr_check(rd->argintregs[0]);
3200
3201                                 if (lm == NULL) {
3202                                         unresolved_method *um = iptr->target;
3203
3204                                         codegen_addpatchref(cd, cd->mcodeptr,
3205                                                                                 PATCHER_invokevirtual, um, 0);
3206
3207                                         if (opt_showdisassemble) {
3208                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3209                                         }
3210
3211                                         s1 = 0;
3212                                         d = um->methodref->parseddesc.md->returntype.type;
3213
3214                                 } else {
3215                                         s1 = OFFSET(vftbl_t, table[0]) +
3216                                                 sizeof(methodptr) * lm->vftblindex;
3217                                         d = lm->parseddesc->returntype.type;
3218                                 }
3219
3220                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3221                                                                            OFFSET(java_objectheader, vftbl),
3222                                                                            REG_ITMP2);
3223                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3224                                 M_CALL(REG_ITMP1);
3225                                 break;
3226
3227                         case ICMD_INVOKEINTERFACE:
3228                                 gen_nullptr_check(rd->argintregs[0]);
3229
3230                                 if (lm == NULL) {
3231                                         unresolved_method *um = iptr->target;
3232
3233                                         codegen_addpatchref(cd, cd->mcodeptr,
3234                                                                                 PATCHER_invokeinterface, um, 0);
3235
3236                                         if (opt_showdisassemble) {
3237                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3238                                         }
3239
3240                                         s1 = 0;
3241                                         s2 = 0;
3242                                         d = um->methodref->parseddesc.md->returntype.type;
3243
3244                                 } else {
3245                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3246                                                 sizeof(methodptr) * lm->class->index;
3247
3248                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3249
3250                                         d = lm->parseddesc->returntype.type;
3251                                 }
3252
3253                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3254                                           OFFSET(java_objectheader, vftbl));
3255                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3256                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3257                                 M_CALL(REG_ITMP1);
3258                                 break;
3259                         }
3260
3261                         /* d contains return type */
3262
3263                         if (d != TYPE_VOID) {
3264                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3265                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3266                                         M_INTMOVE(REG_RESULT, s1);
3267                                         store_reg_to_var_int(iptr->dst, s1);
3268                                 } else {
3269                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3270                                         M_FLTMOVE(REG_FRESULT, s1);
3271                                         store_reg_to_var_flt(iptr->dst, s1);
3272                                 }
3273                         }
3274                         break;
3275
3276
3277                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3278
3279                                       /* op1:   0 == array, 1 == class                */
3280                                       /* val.a: (classinfo *) superclass              */
3281
3282                         /*  superclass is an interface:
3283                          *
3284                          *  OK if ((sub == NULL) ||
3285                          *         (sub->vftbl->interfacetablelength > super->index) &&
3286                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3287                          *
3288                          *  superclass is a class:
3289                          *
3290                          *  OK if ((sub == NULL) || (0
3291                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3292                          *         super->vftbl->diffval));
3293                          */
3294
3295                         if (iptr->op1 == 1) {
3296                                 /* object type cast-check */
3297
3298                                 classinfo *super;
3299                                 vftbl_t   *supervftbl;
3300                                 s4         superindex;
3301
3302                                 super = (classinfo *) iptr->val.a;
3303
3304                                 if (!super) {
3305                                         superindex = 0;
3306                                         supervftbl = NULL;
3307
3308                                 } else {
3309                                         superindex = super->index;
3310                                         supervftbl = super->vftbl;
3311                                 }
3312
3313 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3314                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3315 #endif
3316                                 var_to_reg_int(s1, src, REG_ITMP1);
3317
3318                                 /* calculate interface checkcast code size */
3319
3320                                 s2 = 3; /* mov_membase_reg */
3321                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3322
3323                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3324                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3325                                         3 /* test */ + 6 /* jcc */;
3326
3327                                 if (!super)
3328                                         s2 += (opt_showdisassemble ? 5 : 0);
3329
3330                                 /* calculate class checkcast code size */
3331
3332                                 s3 = 3; /* mov_membase_reg */
3333                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3334                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3335
3336 #if 0
3337                                 if (s1 != REG_ITMP1) {
3338                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3339                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3340                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3341                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3342                                         a += 3;    /* sub */
3343
3344                                 } else
3345 #endif
3346                                         {
3347                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3348                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3349                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3350                                         }
3351
3352                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3353
3354                                 if (!super)
3355                                         s3 += (opt_showdisassemble ? 5 : 0);
3356
3357                                 /* if class is not resolved, check which code to call */
3358
3359                                 if (!super) {
3360                                         M_TEST(s1);
3361                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3362
3363                                         codegen_addpatchref(cd, cd->mcodeptr,
3364                                                                                 PATCHER_checkcast_instanceof_flags,
3365                                                                                 (constant_classref *) iptr->target, 0);
3366
3367                                         if (opt_showdisassemble) {
3368                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3369                                         }
3370
3371                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3372                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3373                                         M_BEQ(s2 + 5);
3374                                 }
3375
3376                                 /* interface checkcast code */
3377
3378                                 if (!super || (super->flags & ACC_INTERFACE)) {
3379                                         if (super) {
3380                                                 M_TEST(s1);
3381                                                 M_BEQ(s2);
3382                                         }
3383
3384                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3385
3386                                         if (!super) {
3387                                                 codegen_addpatchref(cd, cd->mcodeptr,
3388                                                                                         PATCHER_checkcast_instanceof_interface,
3389                                                                                         (constant_classref *) iptr->target, 0);
3390
3391                                                 if (opt_showdisassemble) {
3392                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3393                                                 }
3394                                         }
3395
3396                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3397                                                                                           OFFSET(vftbl_t, interfacetablelength),
3398                                                                                           REG_ITMP3);
3399                                         /* XXX TWISTI: should this be int arithmetic? */
3400                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3401                                         M_TEST(REG_ITMP3);
3402                                         M_BLE(0);
3403                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3404                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3405                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3406                                                                                          superindex * sizeof(methodptr*),
3407                                                                                          REG_ITMP3);
3408                                         M_TEST(REG_ITMP3);
3409                                         M_BEQ(0);
3410                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3411
3412                                         if (!super)
3413                                                 M_JMP_IMM(s3);
3414                                 }
3415
3416                                 /* class checkcast code */
3417
3418                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3419                                         if (super) {
3420                                                 M_TEST(s1);
3421                                                 M_BEQ(s3);
3422                                         }
3423
3424                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3425
3426                                         if (!super) {
3427                                                 codegen_addpatchref(cd, cd->mcodeptr,
3428                                                                                         PATCHER_checkcast_class,
3429                                                                                         (constant_classref *) iptr->target,
3430                                                                                         0);
3431
3432                                                 if (opt_showdisassemble) {
3433                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3434                                                 }
3435                                         }
3436
3437                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3438 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3439                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3440 #endif
3441                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3442                                                                                           OFFSET(vftbl_t, baseval),
3443                                                                                           REG_ITMP2);
3444                                         /*                                      if (s1 != REG_ITMP1) { */
3445                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3446                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3447                                         /*                                                                                              REG_ITMP1); */
3448                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3449                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3450                                         /*                                                                                              REG_ITMP3); */
3451                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3452                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3453                                         /*  #endif */
3454                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3455
3456                                         /*                                      } else { */
3457                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3458                                                                                           OFFSET(vftbl_t, baseval),
3459                                                                                           REG_ITMP3);
3460                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3461                                         M_MOV_IMM((ptrint) supervftbl, REG_ITMP3);
3462                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3463                                         /*                                      } */
3464 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3465                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3466 #endif
3467                                         M_CMP(REG_ITMP3, REG_ITMP2);
3468                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3469                                         codegen_addxcastrefs(cd, cd->mcodeptr);
3470                                 }
3471                                 d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3472
3473                         } else {
3474                                 /* array type cast-check */
3475
3476                                 var_to_reg_int(s1, src, REG_ITMP1);
3477                                 M_INTMOVE(s1, rd->argintregs[0]);
3478
3479                                 if (iptr->val.a == NULL) {
3480                                         codegen_addpatchref(cd, cd->mcodeptr,
3481                                                                                 PATCHER_builtin_arraycheckcast,
3482                                                                                 (constant_classref *) iptr->target, 0);
3483
3484                                         if (opt_showdisassemble) {
3485                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3486                                         }
3487                                 }
3488
3489                                 M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3490                                 M_MOV_IMM((ptrint) BUILTIN_arraycheckcast, REG_ITMP1);
3491                                 M_CALL(REG_ITMP1);
3492                                 M_TEST(REG_RESULT);
3493                                 M_BEQ(0);
3494                                 codegen_addxcastrefs(cd, cd->mcodeptr);
3495
3496                                 var_to_reg_int(s1, src, REG_ITMP1);
3497                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3498                         }
3499                         M_INTMOVE(s1, d);
3500                         store_reg_to_var_int(iptr->dst, d);
3501                         break;
3502
3503                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3504
3505                                       /* op1:   0 == array, 1 == class                */
3506                                       /* val.a: (classinfo *) superclass              */
3507
3508                         /*  superclass is an interface:
3509                          *
3510                          *  return (sub != NULL) &&
3511                          *         (sub->vftbl->interfacetablelength > super->index) &&
3512                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3513                          *
3514                          *  superclass is a class:
3515                          *
3516                          *  return ((sub != NULL) && (0
3517                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3518                          *          super->vftbl->diffvall));
3519                          */
3520
3521                         {
3522                         classinfo *super;
3523                         vftbl_t   *supervftbl;
3524                         s4         superindex;
3525
3526                         super = (classinfo *) iptr->val.a;
3527
3528                         if (!super) {
3529                                 superindex = 0;
3530                                 supervftbl = NULL;
3531
3532                         } else {
3533                                 superindex = super->index;
3534                                 supervftbl = super->vftbl;
3535                         }
3536
3537 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3538             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3539 #endif
3540
3541                         var_to_reg_int(s1, src, REG_ITMP1);
3542                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3543                         if (s1 == d) {
3544                                 M_INTMOVE(s1, REG_ITMP1);
3545                                 s1 = REG_ITMP1;
3546                         }
3547
3548                         /* calculate interface instanceof code size */
3549
3550                         s2 = 3; /* mov_membase_reg */
3551                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3552                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3553                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3554                                 3 /* test */ + 4 /* setcc */;
3555
3556                         if (!super)
3557                                 s2 += (opt_showdisassemble ? 5 : 0);
3558
3559                         /* calculate class instanceof code size */
3560
3561                         s3 = 3; /* mov_membase_reg */
3562                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3563                         s3 += 10; /* mov_imm_reg */
3564                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3565                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3566                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3567                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3568                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3569                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3570                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3571
3572                         if (!super)
3573                                 s3 += (opt_showdisassemble ? 5 : 0);
3574
3575                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3576
3577                         /* if class is not resolved, check which code to call */
3578
3579                         if (!super) {
3580                                 x86_64_test_reg_reg(cd, s1, s1);
3581                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3582                                                                                          7 + 6 + s2 + 5 + s3));
3583
3584                                 codegen_addpatchref(cd, cd->mcodeptr,
3585                                                                         PATCHER_checkcast_instanceof_flags,
3586                                                                         (constant_classref *) iptr->target, 0);
3587
3588                                 if (opt_showdisassemble) {
3589                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3590                                 }
3591
3592                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3593                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3594                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3595                         }
3596
3597                         /* interface instanceof code */
3598
3599                         if (!super || (super->flags & ACC_INTERFACE)) {
3600                                 if (super) {
3601                                         x86_64_test_reg_reg(cd, s1, s1);
3602                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3603                                 }
3604
3605                                 x86_64_mov_membase_reg(cd, s1,
3606                                                                            OFFSET(java_objectheader, vftbl),
3607                                                                            REG_ITMP1);
3608                                 if (!super) {
3609                                         codegen_addpatchref(cd, cd->mcodeptr,
3610                                                                                 PATCHER_checkcast_instanceof_interface,
3611                                                                                 (constant_classref *) iptr->target, 0);
3612
3613                                         if (opt_showdisassemble) {
3614                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3615                                         }
3616                                 }
3617
3618                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3619                                                                                   OFFSET(vftbl_t, interfacetablelength),
3620                                                                                   REG_ITMP3);
3621                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3622                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3623
3624                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3625
3626                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3627                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3628                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3629                                                                                  superindex * sizeof(methodptr*),
3630                                                                                  REG_ITMP1);
3631                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3632                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3633
3634                                 if (!super)
3635                                         x86_64_jmp_imm(cd, s3);
3636                         }
3637
3638                         /* class instanceof code */
3639
3640                         if (!super || !(super->flags & ACC_INTERFACE)) {
3641                                 if (super) {
3642                                         x86_64_test_reg_reg(cd, s1, s1);
3643                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3644                                 }
3645
3646                                 x86_64_mov_membase_reg(cd, s1,
3647                                                                            OFFSET(java_objectheader, vftbl),
3648                                                                            REG_ITMP1);
3649
3650                                 if (!super) {
3651                                         codegen_addpatchref(cd, cd->mcodeptr,
3652                                                                                 PATCHER_instanceof_class,
3653                                                                                 (constant_classref *) iptr->target, 0);
3654
3655                                         if (opt_showdisassemble) {
3656                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3657                                         }
3658                                 }
3659
3660                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3661 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3662                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3663 #endif
3664                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3665                                                                                 OFFSET(vftbl_t, baseval),
3666                                                                                 REG_ITMP1);
3667                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3668                                                                                 OFFSET(vftbl_t, diffval),
3669                                                                                 REG_ITMP3);
3670                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3671                                                                                 OFFSET(vftbl_t, baseval),
3672                                                                                 REG_ITMP2);
3673 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3674                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3675 #endif
3676                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3677                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3678                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3679                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3680                         }
3681                         store_reg_to_var_int(iptr->dst, d);
3682                         }
3683                         break;
3684
3685                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3686                                          /* op1 = dimension, val.a = class            */
3687
3688                         /* check for negative sizes and copy sizes to stack if necessary  */
3689
3690                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3691
3692                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3693                                 /* copy SAVEDVAR sizes to stack */
3694
3695                                 if (src->varkind != ARGVAR) {
3696                                         var_to_reg_int(s2, src, REG_ITMP1);
3697                                         M_LST(s2, REG_SP, s1 * 8);
3698                                 }
3699                         }
3700
3701                         /* is a patcher function set? */
3702
3703                         if (iptr->val.a == NULL) {
3704                                 codegen_addpatchref(cd, cd->mcodeptr,
3705                                                                         PATCHER_builtin_multianewarray,
3706                                                                         (constant_classref *) iptr->target, 0);
3707
3708                                 if (opt_showdisassemble) {
3709                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3710                                 }
3711
3712                                 a = 0;
3713
3714                         } else {
3715                                 a = (ptrint) iptr->val.a;
3716                         }
3717
3718                         /* a0 = dimension count */
3719
3720                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3721
3722                         /* a1 = arrayvftbl */
3723
3724                         M_MOV_IMM((ptrint) iptr->val.a, rd->argintregs[1]);
3725
3726                         /* a2 = pointer to dimensions = stack pointer */
3727
3728                         M_MOV(REG_SP, rd->argintregs[2]);
3729
3730                         M_MOV_IMM((ptrint) BUILTIN_multianewarray, REG_ITMP1);
3731                         M_CALL(REG_ITMP1);
3732
3733                         /* check for exception before result assignment */
3734
3735                         M_TEST(REG_RESULT);
3736                         M_BEQ(0);
3737                         codegen_addxexceptionrefs(cd, cd->mcodeptr);
3738
3739                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3740                         M_INTMOVE(REG_RESULT, s1);
3741                         store_reg_to_var_int(iptr->dst, s1);
3742                         break;
3743
3744                 default:
3745                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3746                         return false;
3747         } /* switch */
3748
3749         } /* for instruction */
3750
3751         /* copy values to interface registers */
3752
3753         src = bptr->outstack;
3754         len = bptr->outdepth;
3755         MCODECHECK(512);
3756 #if defined(ENABLE_LSRA)
3757         if (!opt_lsra)
3758 #endif
3759         while (src) {
3760                 len--;
3761                 if ((src->varkind != STACKVAR)) {
3762                         s2 = src->type;
3763                         if (IS_FLT_DBL_TYPE(s2)) {
3764                                 var_to_reg_flt(s1, src, REG_FTMP1);
3765                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3766                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3767
3768                                 } else {
3769                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3770                                 }
3771
3772                         } else {
3773                                 var_to_reg_int(s1, src, REG_ITMP1);
3774                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3775                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3776
3777                                 } else {
3778                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3779                                 }
3780                         }
3781                 }
3782                 src = src->prev;
3783         }
3784
3785         /* At the end of a basic block we may have to append some nops,
3786            because the patcher stub calling code might be longer than the
3787            actual instruction. So codepatching does not change the
3788            following block unintentionally. */
3789
3790         if (cd->mcodeptr < cd->lastmcodeptr) {
3791                 while (cd->mcodeptr < cd->lastmcodeptr) {
3792                         M_NOP;
3793                 }
3794         }
3795
3796         } /* if (bptr -> flags >= BBREACHED) */
3797         } /* for basic block */
3798
3799         dseg_createlinenumbertable(cd);
3800
3801         {
3802
3803         u1        *xcodeptr;
3804         branchref *bref;
3805
3806         /* generate ArithmeticException stubs */
3807
3808         xcodeptr = NULL;
3809
3810         for (bref = cd->xdivrefs; bref != NULL; bref = bref->next) {
3811                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3812                                   bref->branchpos,
3813                                                   cd->mcodeptr - cd->mcodebase);
3814
3815                 MCODECHECK(512);
3816
3817                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3818                 dseg_adddata(cd, cd->mcodeptr);
3819                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3820
3821                 if (xcodeptr != NULL) {
3822                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3823
3824                 } else {
3825                         xcodeptr = cd->mcodeptr;
3826
3827                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3828                         M_MOV(REG_SP, rd->argintregs[1]);
3829                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3830                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3831
3832                         M_ASUB_IMM(2 * 8, REG_SP);
3833                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3834
3835                         M_MOV_IMM((ptrint) stacktrace_inline_arithmeticexception,
3836                                           REG_ITMP3);
3837                         M_CALL(REG_ITMP3);
3838
3839                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3840                         M_AADD_IMM(2 * 8, REG_SP);
3841
3842                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3843                         M_JMP(REG_ITMP3);
3844                 }
3845         }
3846
3847         /* generate ArrayIndexOutOfBoundsException stubs */
3848
3849         xcodeptr = NULL;
3850
3851         for (bref = cd->xboundrefs; bref != NULL; bref = bref->next) {
3852                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3853                                   bref->branchpos,
3854                                                   cd->mcodeptr - cd->mcodebase);
3855
3856                 MCODECHECK(512);
3857
3858                 /* move index register into REG_ITMP1 */
3859
3860                 M_MOV(bref->reg, REG_ITMP1);                              /* 3 bytes  */
3861
3862                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3863                 dseg_adddata(cd, cd->mcodeptr);
3864                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3865
3866                 if (xcodeptr != NULL) {
3867                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3868
3869                 } else {
3870                         xcodeptr = cd->mcodeptr;
3871
3872                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3873                         M_MOV(REG_SP, rd->argintregs[1]);
3874                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3875                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3876                         M_MOV(REG_ITMP1, rd->argintregs[4]);
3877
3878                         M_ASUB_IMM(2 * 8, REG_SP);
3879                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3880
3881                         M_MOV_IMM((ptrint) stacktrace_inline_arrayindexoutofboundsexception,
3882                                           REG_ITMP3);
3883                         M_CALL(REG_ITMP3);
3884
3885                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3886                         M_AADD_IMM(2 * 8, REG_SP);
3887
3888                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3889                         M_JMP(REG_ITMP3);
3890                 }
3891         }
3892
3893         /* generate ArrayStoreException stubs */
3894
3895         xcodeptr = NULL;
3896
3897         for (bref = cd->xstorerefs; bref != NULL; bref = bref->next) {
3898                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3899                                   bref->branchpos,
3900                                                   cd->mcodeptr - cd->mcodebase);
3901
3902                 MCODECHECK(512);
3903
3904                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3905                 dseg_adddata(cd, cd->mcodeptr);
3906                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3907
3908                 if (xcodeptr != NULL) {
3909                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3910
3911                 } else {
3912                         xcodeptr = cd->mcodeptr;
3913
3914                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3915                         M_MOV(REG_SP, rd->argintregs[1]);
3916                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3917                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3918
3919                         M_ASUB_IMM(2 * 8, REG_SP);
3920                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3921
3922                         M_MOV_IMM((ptrint) stacktrace_inline_arraystoreexception,
3923                                           REG_ITMP3);
3924                         M_CALL(REG_ITMP3);
3925
3926                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3927                         M_AADD_IMM(2 * 8, REG_SP);
3928
3929                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3930                         M_JMP(REG_ITMP3);
3931                 }
3932         }
3933
3934         /* generate ClassCastException stubs */
3935
3936         xcodeptr = NULL;
3937
3938         for (bref = cd->xcastrefs; bref != NULL; bref = bref->next) {
3939                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3940                                   bref->branchpos,
3941                                                   cd->mcodeptr - cd->mcodebase);
3942
3943                 MCODECHECK(512);
3944
3945                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3946                 dseg_adddata(cd, cd->mcodeptr);
3947                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3948
3949                 if (xcodeptr != NULL) {
3950                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3951
3952                 } else {
3953                         xcodeptr = cd->mcodeptr;
3954
3955                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3956                         M_MOV(REG_SP, rd->argintregs[1]);
3957                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3958                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3959
3960                         M_ASUB_IMM(2 * 8, REG_SP);
3961                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3962
3963                         M_MOV_IMM((ptrint) stacktrace_inline_classcastexception, REG_ITMP3);
3964                         M_CALL(REG_ITMP3);
3965
3966                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3967                         M_AADD_IMM(2 * 8, REG_SP);
3968
3969                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
3970                         M_JMP(REG_ITMP3);
3971                 }
3972         }
3973
3974         /* generate NullpointerException stubs */
3975
3976         xcodeptr = NULL;
3977
3978         for (bref = cd->xnullrefs; bref != NULL; bref = bref->next) {
3979                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
3980                                   bref->branchpos,
3981                                                   cd->mcodeptr - cd->mcodebase);
3982
3983                 MCODECHECK(512);
3984
3985                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
3986                 dseg_adddata(cd, cd->mcodeptr);
3987                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
3988
3989                 if (xcodeptr != NULL) {
3990                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
3991
3992                 } else {
3993                         xcodeptr = cd->mcodeptr;
3994
3995                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3996                         M_MOV(REG_SP, rd->argintregs[1]);
3997                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3998                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3999
4000                         M_ASUB_IMM(2 * 8, REG_SP);
4001                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4002
4003                         M_MOV_IMM((ptrint) stacktrace_inline_nullpointerexception,
4004                                           REG_ITMP3);
4005                         M_CALL(REG_ITMP3);
4006
4007                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4008                         M_AADD_IMM(2 * 8, REG_SP);
4009
4010                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4011                         M_JMP(REG_ITMP3);
4012                 }
4013         }
4014
4015         /* generate exception check stubs */
4016
4017         xcodeptr = NULL;
4018
4019         for (bref = cd->xexceptionrefs; bref != NULL; bref = bref->next) {
4020                 gen_resolvebranch(cd->mcodebase + bref->branchpos,
4021                                   bref->branchpos,
4022                                                   cd->mcodeptr - cd->mcodebase);
4023
4024                 MCODECHECK(512);
4025
4026                 M_MOV_IMM(0, REG_ITMP2_XPC);                              /* 10 bytes */
4027                 dseg_adddata(cd, cd->mcodeptr);
4028                 M_AADD_IMM32(bref->branchpos - 6, REG_ITMP2_XPC);         /* 7 bytes  */
4029
4030                 if (xcodeptr != NULL) {
4031                         M_JMP_IMM(xcodeptr - cd->mcodeptr - 5);
4032
4033                 } else {
4034                         xcodeptr = cd->mcodeptr;
4035
4036                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
4037                         M_MOV(REG_SP, rd->argintregs[1]);
4038                         M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
4039                         M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
4040
4041                         M_ASUB_IMM(2 * 8, REG_SP);
4042                         M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
4043
4044                         M_MOV_IMM((ptrint) stacktrace_inline_fillInStackTrace, REG_ITMP3);
4045                         M_CALL(REG_ITMP3);
4046
4047                         M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
4048                         M_AADD_IMM(2 * 8, REG_SP);
4049
4050                         M_MOV_IMM((ptrint) asm_handle_exception, REG_ITMP3);
4051                         M_JMP(REG_ITMP3);
4052                 }
4053         }
4054
4055         /* generate code patching stub call code */
4056
4057         {
4058                 patchref    *pref;
4059                 codegendata *tmpcd;
4060                 ptrint       mcode;
4061
4062                 tmpcd = DNEW(codegendata);
4063
4064                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4065                         /* check size of code segment */
4066
4067                         MCODECHECK(512);
4068
4069                         /* Get machine code which is patched back in later. A             */
4070                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4071
4072                         xcodeptr = cd->mcodebase + pref->branchpos;
4073                         mcode = *((ptrint *) xcodeptr);
4074
4075                         /* patch in `call rel32' to call the following code               */
4076
4077                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4078                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4079
4080                         /* move pointer to java_objectheader onto stack */
4081
4082 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4083                         /* create a virtual java_objectheader */
4084
4085                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4086                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
4087
4088                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
4089                         M_PUSH(REG_ITMP3);
4090 #else
4091                         M_PUSH_IMM(0);
4092 #endif
4093
4094                         /* move machine code bytes and classinfo pointer into registers */
4095
4096                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4097                         M_PUSH(REG_ITMP3);
4098                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4099                         M_PUSH(REG_ITMP3);
4100                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4101                         M_PUSH(REG_ITMP3);
4102
4103                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4104                         M_PUSH(REG_ITMP3);
4105
4106                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4107                         M_JMP(REG_ITMP3);
4108                 }
4109         }
4110         }
4111
4112         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4113
4114         /* everything's ok */
4115
4116         return true;
4117 }
4118
4119
4120 /* createcompilerstub **********************************************************
4121
4122    Creates a stub routine which calls the compiler.
4123
4124 *******************************************************************************/
4125
4126 #define COMPILERSTUB_SIZE    23
4127
4128 u1 *createcompilerstub(methodinfo *m)
4129 {
4130         u1          *s;                     /* memory to hold the stub            */
4131         codegendata *cd;
4132         s4           dumpsize;
4133
4134         s = CNEW(u1, COMPILERSTUB_SIZE);
4135
4136         /* mark start of dump memory area */
4137
4138         dumpsize = dump_size();
4139
4140         cd = DNEW(codegendata);
4141         cd->mcodeptr = s;
4142
4143         /* code for the stub */
4144
4145         M_MOV_IMM((ptrint) m, REG_ITMP1);   /* pass method to compiler            */
4146         M_MOV_IMM((ptrint) asm_call_jit_compiler, REG_ITMP3);
4147         M_JMP(REG_ITMP3);
4148
4149 #if defined(ENABLE_STATISTICS)
4150         if (opt_stat)
4151                 count_cstub_len += COMPILERSTUB_SIZE;
4152 #endif
4153
4154         /* release dump area */
4155
4156         dump_release(dumpsize);
4157
4158         return s;
4159 }
4160
4161
4162 /* createnativestub ************************************************************
4163
4164    Creates a stub routine which calls a native method.
4165
4166 *******************************************************************************/
4167
4168 u1 *createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4169                                          registerdata *rd, methoddesc *nmd)
4170 {
4171         methoddesc *md;
4172         s4          stackframesize;         /* size of stackframe if needed       */
4173         s4          nativeparams;
4174         s4          i, j;                   /* count variables                    */
4175         s4          t;
4176         s4          s1, s2;
4177
4178         /* initialize variables */
4179
4180         md = m->parseddesc;
4181         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4182
4183
4184         /* calculate stack frame size */
4185
4186         stackframesize =
4187                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4188                 sizeof(localref_table) / SIZEOF_VOID_P +
4189                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4190                 nmd->memuse;
4191
4192         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4193                 stackframesize++;
4194
4195
4196         /* create method header */
4197
4198         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4199         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4200         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4201         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4202         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4203         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4204         (void) dseg_addlinenumbertablesize(cd);
4205         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4206
4207
4208         /* initialize mcode variables */
4209
4210         cd->mcodeptr = (u1 *) cd->mcodebase;
4211         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4212
4213
4214         /* generate stub code */
4215
4216         M_ASUB_IMM(stackframesize * 8, REG_SP);
4217
4218         if (runverbose) {
4219                 /* save integer and float argument registers */
4220
4221                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4222                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4223                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4224
4225                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4226                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4227                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4228
4229                 /* show integer hex code for float arguments */
4230
4231                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4232                         /* if the paramtype is a float, we have to right shift all
4233                            following integer registers */
4234
4235                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4236                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4237                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4238
4239                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4240                                 j++;
4241                         }
4242                 }
4243
4244                 M_MOV_IMM((ptrint) m, REG_ITMP1);
4245                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4246                 M_MOV_IMM((ptrint) builtin_trace_args, REG_ITMP1);
4247                 M_CALL(REG_ITMP1);
4248
4249                 /* restore integer and float argument registers */
4250
4251                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4252                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4253                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4254
4255                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4256                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4257                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4258         }
4259
4260
4261         /* get function address (this must happen before the stackframeinfo) */
4262
4263 #if !defined(ENABLE_STATICVM)
4264         if (f == NULL) {
4265                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4266
4267                 if (opt_showdisassemble) {
4268                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4269                 }
4270         }
4271 #endif
4272
4273         M_MOV_IMM((ptrint) f, REG_ITMP3);
4274
4275
4276         /* save integer and float argument registers */
4277
4278         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4279                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4280                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4281
4282         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4283                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4284                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4285
4286         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4287
4288         /* create dynamic stack info */
4289
4290         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4291         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4292         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4293         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4294         M_MOV_IMM((ptrint) codegen_start_native_call, REG_ITMP1);
4295         M_CALL(REG_ITMP1);
4296
4297 #if defined(ENABLE_STATISTICS)
4298         if (opt_stat) {
4299                 M_MOV_IMM((ptrint) nativeinvokation, REG_ITMP1);
4300                 M_CALL(REG_ITMP1);
4301         }
4302 #endif
4303
4304         /* restore integer and float argument registers */
4305
4306         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4307                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4308                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4309
4310         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4311                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4312                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4313
4314         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4315
4316
4317         /* copy or spill arguments to new locations */
4318
4319         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4320                 t = md->paramtypes[i].type;
4321
4322                 if (IS_INT_LNG_TYPE(t)) {
4323                         if (!md->params[i].inmemory) {
4324                                 s1 = rd->argintregs[md->params[i].regoff];
4325
4326                                 if (!nmd->params[j].inmemory) {
4327                                         s2 = rd->argintregs[nmd->params[j].regoff];
4328                                         M_INTMOVE(s1, s2);
4329
4330                                 } else {
4331                                         s2 = nmd->params[j].regoff;
4332                                         M_LST(s1, REG_SP, s2 * 8);
4333                                 }
4334
4335                         } else {
4336                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4337                                 s2 = nmd->params[j].regoff;
4338                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4339                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4340                         }
4341
4342                 } else {
4343                         /* We only copy spilled float arguments, as the float argument    */
4344                         /* registers keep unchanged.                                      */
4345
4346                         if (md->params[i].inmemory) {
4347                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4348                                 s2 = nmd->params[j].regoff;
4349                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4350                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4351                         }
4352                 }
4353         }
4354
4355         /* put class into second argument register */
4356
4357         if (m->flags & ACC_STATIC)
4358                 M_MOV_IMM((ptrint) m->class, rd->argintregs[1]);
4359
4360         /* put env into first argument register */
4361
4362         M_MOV_IMM((ptrint) &env, rd->argintregs[0]);
4363
4364         /* do the native function call */
4365
4366         M_CALL(REG_ITMP3);
4367
4368         /* save return value */
4369
4370         if (md->returntype.type != TYPE_VOID) {
4371                 if (IS_INT_LNG_TYPE(md->returntype.type))
4372                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4373                 else
4374                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4375         }
4376
4377         /* remove native stackframe info */
4378
4379         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4380         M_MOV_IMM((ptrint) codegen_finish_native_call, REG_ITMP1);
4381         M_CALL(REG_ITMP1);
4382
4383         /* generate call trace */
4384
4385         if (runverbose) {
4386                 /* just restore the value we need, don't care about the other */
4387
4388                 if (md->returntype.type != TYPE_VOID) {
4389                         if (IS_INT_LNG_TYPE(md->returntype.type))
4390                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4391                         else
4392                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4393                 }
4394
4395                 M_MOV_IMM((ptrint) m, rd->argintregs[0]);
4396                 M_MOV(REG_RESULT, rd->argintregs[1]);
4397                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4398                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4399
4400                 M_MOV_IMM((ptrint) builtin_displaymethodstop, REG_ITMP1);
4401                 M_CALL(REG_ITMP1);
4402         }
4403
4404         /* check for exception */
4405
4406 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4407         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4408         M_CALL(REG_ITMP3);
4409 #else
4410         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_RESULT);
4411 #endif
4412         M_ALD(REG_ITMP2, REG_RESULT, 0);
4413
4414         /* restore return value */
4415
4416         if (md->returntype.type != TYPE_VOID) {
4417                 if (IS_INT_LNG_TYPE(md->returntype.type))
4418                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4419                 else
4420                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4421         }
4422
4423         /* test for exception */
4424
4425         M_TEST(REG_ITMP2);
4426         M_BNE(7 + 1);
4427
4428         /* remove stackframe */
4429
4430         M_AADD_IMM(stackframesize * 8, REG_SP);
4431         M_RET;
4432
4433
4434         /* handle exception */
4435
4436 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4437         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4438         M_MOV_IMM((ptrint) builtin_get_exceptionptrptr, REG_ITMP3);
4439         M_CALL(REG_ITMP3);
4440         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4441         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4442 #else
4443         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4444         M_MOV_IMM((ptrint) &_no_threads_exceptionptr, REG_ITMP3);
4445         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4446 #endif
4447
4448         /* remove stackframe */
4449
4450         M_AADD_IMM(stackframesize * 8, REG_SP);
4451
4452         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4453         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4454
4455         M_MOV_IMM((ptrint) asm_handle_nat_exception, REG_ITMP3);
4456         M_JMP(REG_ITMP3);
4457
4458
4459         /* process patcher calls **************************************************/
4460
4461         {
4462                 u1          *xcodeptr;
4463                 patchref    *pref;
4464                 codegendata *tmpcd;
4465                 ptrint       mcode;
4466 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4467                 s4           disp;
4468 #endif
4469
4470                 tmpcd = DNEW(codegendata);
4471
4472                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4473                         /* Get machine code which is patched back in later. A             */
4474                         /* `call rel32' is 5 bytes long (but read 8 bytes).               */
4475
4476                         xcodeptr = cd->mcodebase + pref->branchpos;
4477                         mcode = *((ptrint *) xcodeptr);
4478
4479                         /* patch in `call rel32' to call the following code               */
4480
4481                         tmpcd->mcodeptr = xcodeptr;     /* set dummy mcode pointer        */
4482                         x86_64_call_imm(tmpcd, cd->mcodeptr - (xcodeptr + 5));
4483
4484                         /* move pointer to java_objectheader onto stack */
4485
4486 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4487                         /* create a virtual java_objectheader */
4488
4489                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4490                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4491
4492                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4493                         M_PUSH(REG_ITMP3);
4494 #else
4495                         M_PUSH_IMM(0);
4496 #endif
4497
4498                         /* move machine code bytes and classinfo pointer into registers */
4499
4500                         M_MOV_IMM((ptrint) mcode, REG_ITMP3);
4501                         M_PUSH(REG_ITMP3);
4502                         M_MOV_IMM((ptrint) pref->ref, REG_ITMP3);
4503                         M_PUSH(REG_ITMP3);
4504                         M_MOV_IMM((ptrint) pref->disp, REG_ITMP3);
4505                         M_PUSH(REG_ITMP3);
4506
4507                         M_MOV_IMM((ptrint) pref->patcher, REG_ITMP3);
4508                         M_PUSH(REG_ITMP3);
4509
4510                         M_MOV_IMM((ptrint) asm_wrapper_patcher, REG_ITMP3);
4511                         M_JMP(REG_ITMP3);
4512                 }
4513         }
4514
4515         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4516
4517         return m->entrypoint;
4518 }
4519
4520
4521 /*
4522  * These are local overrides for various environment variables in Emacs.
4523  * Please do not remove this and leave it at the end of the file, where
4524  * Emacs will automagically detect them.
4525  * ---------------------------------------------------------------------
4526  * Local variables:
4527  * mode: c
4528  * indent-tabs-mode: t
4529  * c-basic-offset: 4
4530  * tab-width: 4
4531  * End:
4532  */