src/vm/jit/x86_64/codegen.c

   1 /* src/vm/jit/x86_64/codegen.c - machine code generator for x86_64
   2
   3    Copyright (C) 1996-2005, 2006 R. Grafl, A. Krall, C. Kruegel,
   4    C. Oates, R. Obermaisser, M. Platter, M. Probst, S. Ring,
   5    E. Steiner, C. Thalinger, D. Thuernbeck, P. Tomsich, C. Ullrich,
   6    J. Wenninger, Institut f. Computersprachen - TU Wien
   7
   8    This file is part of CACAO.
   9
  10    This program is free software; you can redistribute it and/or
  11    modify it under the terms of the GNU General Public License as
  12    published by the Free Software Foundation; either version 2, or (at
  13    your option) any later version.
  14
  15    This program is distributed in the hope that it will be useful, but
  16    WITHOUT ANY WARRANTY; without even the implied warranty of
  17    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18    General Public License for more details.
  19
  20    You should have received a copy of the GNU General Public License
  21    along with this program; if not, write to the Free Software
  22    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  23    02110-1301, USA.
  24
  25    Contact: cacao@cacaojvm.org
  26
  27    Authors: Andreas Krall
  28             Christian Thalinger
  29
  30    Changes: Christian Ullrich
  31             Edwin Steiner
  32
  33    $Id: codegen.c 4690 2006-03-27 11:37:46Z twisti $
  34
  35 */
  36
  37
  38 #include "config.h"
  39
  40 #include <assert.h>
  41 #include <stdio.h>
  42
  43 #include "vm/types.h"
  44
  45 #include "md.h"
  46 #include "md-abi.h"
  47
  48 #include "vm/jit/x86_64/arch.h"
  49 #include "vm/jit/x86_64/codegen.h"
  50 #include "vm/jit/x86_64/emitfuncs.h"
  51
  52 #include "native/native.h"
  53 #include "vm/builtin.h"
  54 #include "vm/exceptions.h"
  55 #include "vm/global.h"
  56 #include "vm/loader.h"
  57 #include "vm/options.h"
  58 #include "vm/statistics.h"
  59 #include "vm/stringlocal.h"
  60 #include "vm/vm.h"
  61 #include "vm/jit/asmpart.h"
  62 #include "vm/jit/codegen-common.h"
  63 #include "vm/jit/dseg.h"
  64 #include "vm/jit/jit.h"
  65 #include "vm/jit/methodheader.h"
  66 #include "vm/jit/parse.h"
  67 #include "vm/jit/patcher.h"
  68 #include "vm/jit/reg.h"
  69 #include "vm/jit/replace.h"
  70
  71 #if defined(ENABLE_LSRA)
  72 # include "vm/jit/allocator/lsra.h"
  73 #endif
  74
  75
  76
  77
  78 /* codegen *********************************************************************
  79
  80    Generates machine code.
  81
  82 *******************************************************************************/
  83
  84 bool codegen(methodinfo *m, codegendata *cd, registerdata *rd)
  85 {
  86         s4                  len, s1, s2, s3, d, disp;
  87         u2                  currentline;
  88         ptrint              a;
  89         s4                  parentargs_base;
  90         stackptr            src;
  91         varinfo            *var;
  92         basicblock         *bptr;
  93         instruction        *iptr;
  94         exceptiontable     *ex;
  95         methodinfo         *lm;             /* local methodinfo for ICMD_INVOKE*  */
  96         builtintable_entry *bte;
  97         methoddesc         *md;
  98         rplpoint           *replacementpoint;
  99
 100         /* prevent compiler warnings */
 101
 102         d = 0;
 103         lm = NULL;
 104         bte = NULL;
 105
 106         {
 107         s4 i, p, t, l;
 108         s4 savedregs_num;
 109
 110         savedregs_num = 0;
 111
 112         /* space to save used callee saved registers */
 113
 114         savedregs_num += (INT_SAV_CNT - rd->savintreguse);
 115         savedregs_num += (FLT_SAV_CNT - rd->savfltreguse);
 116
 117         parentargs_base = rd->memuse + savedregs_num;
 118
 119 #if defined(USE_THREADS)
 120         /* space to save argument of monitor_enter */
 121
 122         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 123                 parentargs_base++;
 124 #endif
 125
 126     /* Keep stack of non-leaf functions 16-byte aligned for calls into native */
 127         /* code e.g. libc or jni (alignment problems with movaps).                */
 128
 129         if (!m->isleafmethod || opt_verbosecall)
 130                 parentargs_base |= 0x1;
 131
 132         /* create method header */
 133
 134         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
 135         (void) dseg_adds4(cd, parentargs_base * 8);             /* FrameSize      */
 136
 137 #if defined(USE_THREADS)
 138         /* IsSync contains the offset relative to the stack pointer for the
 139            argument of monitor_exit used in the exception handler. Since the
 140            offset could be zero and give a wrong meaning of the flag it is
 141            offset by one.
 142         */
 143
 144         if (checksync && (m->flags & ACC_SYNCHRONIZED))
 145                 (void) dseg_adds4(cd, (rd->memuse + 1) * 8);        /* IsSync         */
 146         else
 147 #endif
 148                 (void) dseg_adds4(cd, 0);                           /* IsSync         */
 149
 150         (void) dseg_adds4(cd, m->isleafmethod);                 /* IsLeaf         */
 151         (void) dseg_adds4(cd, INT_SAV_CNT - rd->savintreguse);  /* IntSave        */
 152         (void) dseg_adds4(cd, FLT_SAV_CNT - rd->savfltreguse);  /* FltSave        */
 153
 154         (void) dseg_addlinenumbertablesize(cd);
 155
 156         (void) dseg_adds4(cd, cd->exceptiontablelength);        /* ExTableSize    */
 157
 158         /* create exception table */
 159
 160         for (ex = cd->exceptiontable; ex != NULL; ex = ex->down) {
 161                 dseg_addtarget(cd, ex->start);
 162                 dseg_addtarget(cd, ex->end);
 163                 dseg_addtarget(cd, ex->handler);
 164                 (void) dseg_addaddress(cd, ex->catchtype.cls);
 165         }
 166
 167         /* initialize mcode variables */
 168
 169         cd->mcodeptr = (u1 *) cd->mcodebase;
 170         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
 171
 172         /* initialize the last patcher pointer */
 173
 174         cd->lastmcodeptr = cd->mcodeptr;
 175
 176         /* generate method profiling code */
 177
 178         if (opt_prof) {
 179                 /* count frequency */
 180
 181                 M_MOV_IMM(m, REG_ITMP3);
 182                 M_IINC_MEMBASE(REG_ITMP3, OFFSET(methodinfo, frequency));
 183
 184                 PROFILE_CYCLE_START;
 185         }
 186
 187         /* create stack frame (if necessary) */
 188
 189         if (parentargs_base)
 190                 M_ASUB_IMM(parentargs_base * 8, REG_SP);
 191
 192         /* save used callee saved registers */
 193
 194         p = parentargs_base;
 195         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
 196                 p--; M_LST(rd->savintregs[i], REG_SP, p * 8);
 197         }
 198         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
 199                 p--; M_DST(rd->savfltregs[i], REG_SP, p * 8);
 200         }
 201
 202         /* take arguments out of register or stack frame */
 203
 204         md = m->parseddesc;
 205
 206         for (p = 0, l = 0; p < md->paramcount; p++) {
 207                 t = md->paramtypes[p].type;
 208                 var = &(rd->locals[l][t]);
 209                 l++;
 210                 if (IS_2_WORD_TYPE(t))    /* increment local counter for 2 word types */
 211                         l++;
 212                 if (var->type < 0)
 213                         continue;
 214                 s1 = md->params[p].regoff;
 215                 if (IS_INT_LNG_TYPE(t)) {                    /* integer args          */
 216                         s2 = rd->argintregs[s1];
 217                         if (!md->params[p].inmemory) {           /* register arguments    */
 218                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 219                                         M_INTMOVE(s2, var->regoff);
 220
 221                                 } else {                             /* reg arg -> spilled    */
 222                                     M_LST(s2, REG_SP, var->regoff * 8);
 223                                 }
 224
 225                         } else {                                 /* stack arguments       */
 226                                 if (!(var->flags & INMEMORY)) {      /* stack arg -> register */
 227                                         /* + 8 for return address */
 228                                         M_LLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 229
 230                                 } else {                             /* stack arg -> spilled  */
 231                                         var->regoff = parentargs_base + s1 + 1;
 232                                 }
 233                         }
 234
 235                 } else {                                     /* floating args         */
 236                         if (!md->params[p].inmemory) {           /* register arguments    */
 237                                 s2 = rd->argfltregs[s1];
 238                                 if (!(var->flags & INMEMORY)) {      /* reg arg -> register   */
 239                                         M_FLTMOVE(s2, var->regoff);
 240
 241                                 } else {                                         /* reg arg -> spilled    */
 242                                         M_DST(s2, REG_SP, var->regoff * 8);
 243                                 }
 244
 245                         } else {                                 /* stack arguments       */
 246                                 if (!(var->flags & INMEMORY)) {      /* stack-arg -> register */
 247                                         M_DLD(var->regoff, REG_SP, (parentargs_base + s1) * 8 + 8);
 248
 249                                 } else {
 250                                         var->regoff = parentargs_base + s1 + 1;
 251                                 }
 252                         }
 253                 }
 254         }  /* end for */
 255
 256         /* save monitorenter argument */
 257
 258 #if defined(USE_THREADS)
 259         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
 260                 /* stack offset for monitor argument */
 261
 262                 s1 = rd->memuse;
 263
 264                 if (opt_verbosecall) {
 265                         M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 266
 267                         for (p = 0; p < INT_ARG_CNT; p++)
 268                                 M_LST(rd->argintregs[p], REG_SP, p * 8);
 269
 270                         for (p = 0; p < FLT_ARG_CNT; p++)
 271                                 M_DST(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 272
 273                         s1 += INT_ARG_CNT + FLT_ARG_CNT;
 274                 }
 275
 276                 /* decide which monitor enter function to call */
 277
 278                 if (m->flags & ACC_STATIC) {
 279                         M_MOV_IMM(m->class, REG_ITMP1);
 280                         M_AST(REG_ITMP1, REG_SP, s1 * 8);
 281                         M_INTMOVE(REG_ITMP1, rd->argintregs[0]);
 282                         M_MOV_IMM(BUILTIN_staticmonitorenter, REG_ITMP1);
 283                         M_CALL(REG_ITMP1);
 284
 285                 } else {
 286                         M_TEST(rd->argintregs[0]);
 287                         M_BEQ(0);
 288                         codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 289                         M_AST(rd->argintregs[0], REG_SP, s1 * 8);
 290                         M_MOV_IMM(BUILTIN_monitorenter, REG_ITMP1);
 291                         M_CALL(REG_ITMP1);
 292                 }
 293
 294                 if (opt_verbosecall) {
 295                         for (p = 0; p < INT_ARG_CNT; p++)
 296                                 M_LLD(rd->argintregs[p], REG_SP, p * 8);
 297
 298                         for (p = 0; p < FLT_ARG_CNT; p++)
 299                                 M_DLD(rd->argfltregs[p], REG_SP, (INT_ARG_CNT + p) * 8);
 300
 301                         M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT) * 8, REG_SP);
 302                 }
 303         }
 304 #endif
 305
 306 #if !defined(NDEBUG)
 307         /* Copy argument registers to stack and call trace function with
 308            pointer to arguments on stack. */
 309
 310         if (opt_verbosecall) {
 311                 M_LSUB_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 312
 313                 /* save integer argument registers */
 314
 315                 for (p = 0; p < INT_ARG_CNT; p++)
 316                         M_LST(rd->argintregs[p], REG_SP, (1 + p) * 8);
 317
 318                 /* save float argument registers */
 319
 320                 for (p = 0; p < FLT_ARG_CNT; p++)
 321                         M_DST(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 322
 323                 /* save temporary registers for leaf methods */
 324
 325                 if (m->isleafmethod) {
 326                         for (p = 0; p < INT_TMP_CNT; p++)
 327                                 M_LST(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 328
 329                         for (p = 0; p < FLT_TMP_CNT; p++)
 330                                 M_DST(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 331                 }
 332
 333                 /* show integer hex code for float arguments */
 334
 335                 for (p = 0, l = 0; p < md->paramcount && p < INT_ARG_CNT; p++) {
 336                         /* If the paramtype is a float, we have to right shift all
 337                            following integer registers. */
 338
 339                         if (IS_FLT_DBL_TYPE(md->paramtypes[p].type)) {
 340                                 for (s1 = INT_ARG_CNT - 2; s1 >= p; s1--) {
 341                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
 342                                 }
 343
 344                                 x86_64_movd_freg_reg(cd, rd->argfltregs[l], rd->argintregs[p]);
 345                                 l++;
 346                         }
 347                 }
 348
 349                 M_MOV_IMM(m, REG_ITMP2);
 350                 M_AST(REG_ITMP2, REG_SP, 0 * 8);
 351                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
 352                 M_CALL(REG_ITMP1);
 353
 354                 /* restore integer argument registers */
 355
 356                 for (p = 0; p < INT_ARG_CNT; p++)
 357                         M_LLD(rd->argintregs[p], REG_SP, (1 + p) * 8);
 358
 359                 /* restore float argument registers */
 360
 361                 for (p = 0; p < FLT_ARG_CNT; p++)
 362                         M_DLD(rd->argfltregs[p], REG_SP, (1 + INT_ARG_CNT + p) * 8);
 363
 364                 /* restore temporary registers for leaf methods */
 365
 366                 if (m->isleafmethod) {
 367                         for (p = 0; p < INT_TMP_CNT; p++)
 368                                 M_LLD(rd->tmpintregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + p) * 8);
 369
 370                         for (p = 0; p < FLT_TMP_CNT; p++)
 371                                 M_DLD(rd->tmpfltregs[p], REG_SP, (1 + INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + p) * 8);
 372                 }
 373
 374                 M_LADD_IMM((INT_ARG_CNT + FLT_ARG_CNT + INT_TMP_CNT + FLT_TMP_CNT + 1 + 1) * 8, REG_SP);
 375         }
 376 #endif /* !defined(NDEBUG) */
 377
 378         }
 379
 380         /* end of header generation */
 381
 382         replacementpoint = cd->code->rplpoints;
 383
 384         /* walk through all basic blocks */
 385
 386         for (bptr = m->basicblocks; bptr != NULL; bptr = bptr->next) {
 387
 388                 bptr->mpc = (u4) ((u1 *) cd->mcodeptr - cd->mcodebase);
 389
 390                 if (bptr->flags >= BBREACHED) {
 391
 392                         /* branch resolving */
 393
 394                         branchref *bref;
 395                         for (bref = bptr->branchrefs; bref != NULL; bref = bref->next) {
 396                                 gen_resolvebranch((u1 *) cd->mcodebase + bref->branchpos,
 397                                                                   bref->branchpos,
 398                                                                   bptr->mpc);
 399                         }
 400
 401                 /* handle replacement points */
 402
 403                 if (bptr->bitflags & BBFLAG_REPLACEMENT) {
 404                         replacementpoint->pc = (u1*)(ptrint)bptr->mpc; /* will be resolved later */
 405
 406                         replacementpoint++;
 407
 408                         assert(cd->lastmcodeptr <= cd->mcodeptr);
 409                         cd->lastmcodeptr = cd->mcodeptr + 5; /* 5 byte jmp patch */
 410                 }
 411
 412                 /* copy interface registers to their destination */
 413
 414                 src = bptr->instack;
 415                 len = bptr->indepth;
 416                 MCODECHECK(512);
 417
 418                 /* generate basicblock profiling code */
 419
 420                 if (opt_prof_bb) {
 421                         /* count frequency */
 422
 423                         M_MOV_IMM(m->bbfrequency, REG_ITMP2);
 424                         M_IINC_MEMBASE(REG_ITMP2, bptr->debug_nr * 4);
 425
 426                         /* if this is an exception handler, start profiling again */
 427
 428                         if (bptr->type == BBTYPE_EXH)
 429                                 PROFILE_CYCLE_START;
 430                 }
 431
 432 #if defined(ENABLE_LSRA)
 433                 if (opt_lsra) {
 434                         while (src != NULL) {
 435                                 len--;
 436                                 if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 437                                         if (bptr->type == BBTYPE_SBR) {
 438                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 439                                                 if (!(src->flags & INMEMORY))
 440                                                         d= src->regoff;
 441                                                 else
 442                                                         d=REG_ITMP1;
 443                                                 x86_64_pop_reg(cd, d);
 444                                                 store_reg_to_var_int(src, d);
 445
 446                                         } else if (bptr->type == BBTYPE_EXH) {
 447                                                 /*                                      d = reg_of_var(rd, src, REG_ITMP1); */
 448                                                 if (!(src->flags & INMEMORY))
 449                                                         d= src->regoff;
 450                                                 else
 451                                                         d=REG_ITMP1;
 452                                                 M_INTMOVE(REG_ITMP1, d);
 453                                                 store_reg_to_var_int(src, d);
 454                                         }
 455                                 }
 456                                 src = src->prev;
 457                         }
 458
 459                 } else {
 460 #endif
 461
 462                 while (src != NULL) {
 463                         len--;
 464                         if ((len == 0) && (bptr->type != BBTYPE_STD)) {
 465                                 if (bptr->type == BBTYPE_SBR) {
 466                                         d = reg_of_var(rd, src, REG_ITMP1);
 467                                         M_POP(d);
 468                                         store_reg_to_var_int(src, d);
 469
 470                                 } else if (bptr->type == BBTYPE_EXH) {
 471                                         d = reg_of_var(rd, src, REG_ITMP1);
 472                                         M_INTMOVE(REG_ITMP1, d);
 473                                         store_reg_to_var_int(src, d);
 474                                 }
 475
 476                         } else {
 477                                 d = reg_of_var(rd, src, REG_ITMP1);
 478                                 if ((src->varkind != STACKVAR)) {
 479                                         s2 = src->type;
 480                                         if (IS_FLT_DBL_TYPE(s2)) {
 481                                                 s1 = rd->interfaces[len][s2].regoff;
 482
 483                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 484                                                         M_FLTMOVE(s1, d);
 485                                                 else
 486                                                         M_DLD(d, REG_SP, s1 * 8);
 487
 488                                                 store_reg_to_var_flt(src, d);
 489
 490                                         } else {
 491                                                 s1 = rd->interfaces[len][s2].regoff;
 492
 493                                                 if (!(rd->interfaces[len][s2].flags & INMEMORY))
 494                                                         M_INTMOVE(s1, d);
 495                                                 else
 496                                                         M_LLD(d, REG_SP, s1 * 8);
 497
 498                                                 store_reg_to_var_int(src, d);
 499                                         }
 500                                 }
 501                         }
 502                         src = src->prev;
 503                 }
 504 #if defined(ENABLE_LSRA)
 505                 }
 506 #endif
 507                 /* walk through all instructions */
 508
 509                 src = bptr->instack;
 510                 len = bptr->icount;
 511                 currentline = 0;
 512
 513                 for (iptr = bptr->iinstr; len > 0; src = iptr->dst, len--, iptr++) {
 514                         if (iptr->line != currentline) {
 515                                 dseg_addlinenumber(cd, iptr->line, cd->mcodeptr);
 516                                 currentline = iptr->line;
 517                         }
 518
 519                         MCODECHECK(1024);                         /* 1KB should be enough */
 520
 521                         switch (iptr->opc) {
 522                         case ICMD_INLINE_START: /* internal ICMDs                         */
 523                         case ICMD_INLINE_END:
 524                                 break;
 525
 526                         case ICMD_NOP:    /* ...  ==> ...                                 */
 527                                 break;
 528
 529                         case ICMD_CHECKNULL: /* ..., objectref  ==> ..., objectref        */
 530
 531                                 if (src->flags & INMEMORY)
 532                                         M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
 533                                 else
 534                                         M_TEST(src->regoff);
 535                                 M_BEQ(0);
 536                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
 537                                 break;
 538
 539                 /* constant operations ************************************************/
 540
 541                 case ICMD_ICONST:     /* ...  ==> ..., constant                       */
 542                                       /* op1 = 0, val.i = constant                    */
 543
 544                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 545                         if (iptr->val.i == 0)
 546                                 M_CLR(d);
 547                         else
 548                                 M_IMOV_IMM(iptr->val.i, d);
 549                         store_reg_to_var_int(iptr->dst, d);
 550                         break;
 551
 552                 case ICMD_LCONST:     /* ...  ==> ..., constant                       */
 553                                       /* op1 = 0, val.l = constant                    */
 554
 555                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 556                         if (iptr->val.l == 0)
 557                                 M_CLR(d);
 558                         else
 559                                 M_MOV_IMM(iptr->val.l, d);
 560                         store_reg_to_var_int(iptr->dst, d);
 561                         break;
 562
 563                 case ICMD_FCONST:     /* ...  ==> ..., constant                       */
 564                                       /* op1 = 0, val.f = constant                    */
 565
 566                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 567                         disp = dseg_addfloat(cd, iptr->val.f);
 568                         x86_64_movdl_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + ((d > 7) ? 9 : 8)) - (s8) cd->mcodebase) + disp, d);
 569                         store_reg_to_var_flt(iptr->dst, d);
 570                         break;
 571
 572                 case ICMD_DCONST:     /* ...  ==> ..., constant                       */
 573                                       /* op1 = 0, val.d = constant                    */
 574
 575                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 576                         disp = dseg_adddouble(cd, iptr->val.d);
 577                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, d);
 578                         store_reg_to_var_flt(iptr->dst, d);
 579                         break;
 580
 581                 case ICMD_ACONST:     /* ...  ==> ..., constant                       */
 582                                       /* op1 = 0, val.a = constant                    */
 583
 584                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 585
 586                         if ((iptr->target != NULL) && (iptr->val.a == NULL)) {
 587 /*                              PROFILE_CYCLE_STOP; */
 588
 589                                 codegen_addpatchref(cd, cd->mcodeptr,
 590                                                                         PATCHER_aconst,
 591                                                                         (unresolved_class *) iptr->target, 0);
 592
 593                                 if (opt_showdisassemble) {
 594                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
 595                                 }
 596
 597 /*                              PROFILE_CYCLE_START; */
 598
 599                                 M_MOV_IMM(iptr->val.a, d);
 600
 601                         } else {
 602                                 if (iptr->val.a == 0)
 603                                         M_CLR(d);
 604                                 else
 605                                         M_MOV_IMM(iptr->val.a, d);
 606                         }
 607                         store_reg_to_var_int(iptr->dst, d);
 608                         break;
 609
 610
 611                 /* load/store operations **********************************************/
 612
 613                 case ICMD_ILOAD:      /* ...  ==> ..., content of local variable      */
 614                                       /* op1 = local variable                         */
 615
 616                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 617                         if ((iptr->dst->varkind == LOCALVAR) &&
 618                             (iptr->dst->varnum == iptr->op1)) {
 619                                 break;
 620                         }
 621                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 622                         if (var->flags & INMEMORY) {
 623                                 x86_64_movl_membase_reg(cd, REG_SP, var->regoff * 8, d);
 624                                 store_reg_to_var_int(iptr->dst, d);
 625
 626                         } else {
 627                                 if (iptr->dst->flags & INMEMORY) {
 628                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 629
 630                                 } else {
 631                                         M_INTMOVE(var->regoff, d);
 632                                 }
 633                         }
 634                         break;
 635
 636                 case ICMD_LLOAD:      /* ...  ==> ..., content of local variable      */
 637                 case ICMD_ALOAD:      /* op1 = local variable                         */
 638
 639                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
 640                         if ((iptr->dst->varkind == LOCALVAR) &&
 641                             (iptr->dst->varnum == iptr->op1)) {
 642                                 break;
 643                         }
 644                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 645                         if (var->flags & INMEMORY) {
 646                                 x86_64_mov_membase_reg(cd, REG_SP, var->regoff * 8, d);
 647                                 store_reg_to_var_int(iptr->dst, d);
 648
 649                         } else {
 650                                 if (iptr->dst->flags & INMEMORY) {
 651                                         x86_64_mov_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 652
 653                                 } else {
 654                                         M_INTMOVE(var->regoff, d);
 655                                 }
 656                         }
 657                         break;
 658
 659                 case ICMD_FLOAD:      /* ...  ==> ..., content of local variable      */
 660                 case ICMD_DLOAD:      /* op1 = local variable                         */
 661
 662                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
 663                         if ((iptr->dst->varkind == LOCALVAR) &&
 664                             (iptr->dst->varnum == iptr->op1)) {
 665                                 break;
 666                         }
 667                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ILOAD]);
 668                         if (var->flags & INMEMORY) {
 669                                 x86_64_movq_membase_reg(cd, REG_SP, var->regoff * 8, d);
 670                                 store_reg_to_var_flt(iptr->dst, d);
 671
 672                         } else {
 673                                 if (iptr->dst->flags & INMEMORY) {
 674                                         x86_64_movq_reg_membase(cd, var->regoff, REG_SP, iptr->dst->regoff * 8);
 675
 676                                 } else {
 677                                         M_FLTMOVE(var->regoff, d);
 678                                 }
 679                         }
 680                         break;
 681
 682                 case ICMD_ISTORE:     /* ..., value  ==> ...                          */
 683                 case ICMD_LSTORE:     /* op1 = local variable                         */
 684                 case ICMD_ASTORE:
 685
 686                         if ((src->varkind == LOCALVAR) &&
 687                             (src->varnum == iptr->op1)) {
 688                                 break;
 689                         }
 690                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 691                         if (var->flags & INMEMORY) {
 692                                 var_to_reg_int(s1, src, REG_ITMP1);
 693                                 x86_64_mov_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 694
 695                         } else {
 696                                 var_to_reg_int(s1, src, var->regoff);
 697                                 M_INTMOVE(s1, var->regoff);
 698                         }
 699                         break;
 700
 701                 case ICMD_FSTORE:     /* ..., value  ==> ...                          */
 702                 case ICMD_DSTORE:     /* op1 = local variable                         */
 703
 704                         if ((src->varkind == LOCALVAR) &&
 705                             (src->varnum == iptr->op1)) {
 706                                 break;
 707                         }
 708                         var = &(rd->locals[iptr->op1][iptr->opc - ICMD_ISTORE]);
 709                         if (var->flags & INMEMORY) {
 710                                 var_to_reg_flt(s1, src, REG_FTMP1);
 711                                 x86_64_movq_reg_membase(cd, s1, REG_SP, var->regoff * 8);
 712
 713                         } else {
 714                                 var_to_reg_flt(s1, src, var->regoff);
 715                                 M_FLTMOVE(s1, var->regoff);
 716                         }
 717                         break;
 718
 719
 720                 /* pop/dup/swap operations ********************************************/
 721
 722                 /* attention: double and longs are only one entry in CACAO ICMDs      */
 723
 724                 case ICMD_POP:        /* ..., value  ==> ...                          */
 725                 case ICMD_POP2:       /* ..., value, value  ==> ...                   */
 726                         break;
 727
 728                 case ICMD_DUP:        /* ..., a ==> ..., a, a                         */
 729                         M_COPY(src, iptr->dst);
 730                         break;
 731
 732                 case ICMD_DUP_X1:     /* ..., a, b ==> ..., b, a, b                   */
 733
 734                         M_COPY(src,       iptr->dst);
 735                         M_COPY(src->prev, iptr->dst->prev);
 736                         M_COPY(iptr->dst, iptr->dst->prev->prev);
 737                         break;
 738
 739                 case ICMD_DUP_X2:     /* ..., a, b, c ==> ..., c, a, b, c             */
 740
 741                         M_COPY(src,             iptr->dst);
 742                         M_COPY(src->prev,       iptr->dst->prev);
 743                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 744                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 745                         break;
 746
 747                 case ICMD_DUP2:       /* ..., a, b ==> ..., a, b, a, b                */
 748
 749                         M_COPY(src,       iptr->dst);
 750                         M_COPY(src->prev, iptr->dst->prev);
 751                         break;
 752
 753                 case ICMD_DUP2_X1:    /* ..., a, b, c ==> ..., b, c, a, b, c          */
 754
 755                         M_COPY(src,             iptr->dst);
 756                         M_COPY(src->prev,       iptr->dst->prev);
 757                         M_COPY(src->prev->prev, iptr->dst->prev->prev);
 758                         M_COPY(iptr->dst,       iptr->dst->prev->prev->prev);
 759                         M_COPY(iptr->dst->prev, iptr->dst->prev->prev->prev->prev);
 760                         break;
 761
 762                 case ICMD_DUP2_X2:    /* ..., a, b, c, d ==> ..., c, d, a, b, c, d    */
 763
 764                         M_COPY(src,                   iptr->dst);
 765                         M_COPY(src->prev,             iptr->dst->prev);
 766                         M_COPY(src->prev->prev,       iptr->dst->prev->prev);
 767                         M_COPY(src->prev->prev->prev, iptr->dst->prev->prev->prev);
 768                         M_COPY(iptr->dst,             iptr->dst->prev->prev->prev->prev);
 769                         M_COPY(iptr->dst->prev,       iptr->dst->prev->prev->prev->prev->prev);
 770                         break;
 771
 772                 case ICMD_SWAP:       /* ..., a, b ==> ..., b, a                      */
 773
 774                         M_COPY(src,       iptr->dst->prev);
 775                         M_COPY(src->prev, iptr->dst);
 776                         break;
 777
 778
 779                 /* integer operations *************************************************/
 780
 781                 case ICMD_INEG:       /* ..., value  ==> ..., - value                 */
 782
 783                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 784                         if (iptr->dst->flags & INMEMORY) {
 785                                 if (src->flags & INMEMORY) {
 786                                         if (src->regoff == iptr->dst->regoff) {
 787                                                 x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 788
 789                                         } else {
 790                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 791                                                 x86_64_negl_reg(cd, REG_ITMP1);
 792                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 793                                         }
 794
 795                                 } else {
 796                                         x86_64_movl_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 797                                         x86_64_negl_membase(cd, REG_SP, iptr->dst->regoff * 8);
 798                                 }
 799
 800                         } else {
 801                                 if (src->flags & INMEMORY) {
 802                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 803                                         x86_64_negl_reg(cd, d);
 804
 805                                 } else {
 806                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 807                                         x86_64_negl_reg(cd, iptr->dst->regoff);
 808                                 }
 809                         }
 810                         break;
 811
 812                 case ICMD_LNEG:       /* ..., value  ==> ..., - value                 */
 813
 814                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 815                         if (iptr->dst->flags & INMEMORY) {
 816                                 if (src->flags & INMEMORY) {
 817                                         if (src->regoff == iptr->dst->regoff) {
 818                                                 x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 819
 820                                         } else {
 821                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 822                                                 x86_64_neg_reg(cd, REG_ITMP1);
 823                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 824                                         }
 825
 826                                 } else {
 827                                         x86_64_mov_reg_membase(cd, src->regoff, REG_SP, iptr->dst->regoff * 8);
 828                                         x86_64_neg_membase(cd, REG_SP, iptr->dst->regoff * 8);
 829                                 }
 830
 831                         } else {
 832                                 if (src->flags & INMEMORY) {
 833                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
 834                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 835
 836                                 } else {
 837                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
 838                                         x86_64_neg_reg(cd, iptr->dst->regoff);
 839                                 }
 840                         }
 841                         break;
 842
 843                 case ICMD_I2L:        /* ..., value  ==> ..., value                   */
 844
 845                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 846                         if (src->flags & INMEMORY) {
 847                                 x86_64_movslq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 848
 849                         } else {
 850                                 x86_64_movslq_reg_reg(cd, src->regoff, d);
 851                         }
 852                         store_reg_to_var_int(iptr->dst, d);
 853                         break;
 854
 855                 case ICMD_L2I:        /* ..., value  ==> ..., value                   */
 856
 857                         var_to_reg_int(s1, src, REG_ITMP1);
 858                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 859                         M_INTMOVE(s1, d);
 860                         store_reg_to_var_int(iptr->dst, d);
 861                         break;
 862
 863                 case ICMD_INT2BYTE:   /* ..., value  ==> ..., value                   */
 864
 865                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 866                         if (src->flags & INMEMORY) {
 867                                 x86_64_movsbq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 868
 869                         } else {
 870                                 x86_64_movsbq_reg_reg(cd, src->regoff, d);
 871                         }
 872                         store_reg_to_var_int(iptr->dst, d);
 873                         break;
 874
 875                 case ICMD_INT2CHAR:   /* ..., value  ==> ..., value                   */
 876
 877                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 878                         if (src->flags & INMEMORY) {
 879                                 x86_64_movzwq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 880
 881                         } else {
 882                                 x86_64_movzwq_reg_reg(cd, src->regoff, d);
 883                         }
 884                         store_reg_to_var_int(iptr->dst, d);
 885                         break;
 886
 887                 case ICMD_INT2SHORT:  /* ..., value  ==> ..., value                   */
 888
 889                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
 890                         if (src->flags & INMEMORY) {
 891                                 x86_64_movswq_membase_reg(cd, REG_SP, src->regoff * 8, d);
 892
 893                         } else {
 894                                 x86_64_movswq_reg_reg(cd, src->regoff, d);
 895                         }
 896                         store_reg_to_var_int(iptr->dst, d);
 897                         break;
 898
 899
 900                 case ICMD_IADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 901
 902                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 903                         x86_64_emit_ialu(cd, X86_64_ADD, src, iptr);
 904                         break;
 905
 906                 case ICMD_IADDCONST:  /* ..., value  ==> ..., value + constant        */
 907                                       /* val.i = constant                             */
 908
 909                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 910                         x86_64_emit_ialuconst(cd, X86_64_ADD, src, iptr);
 911                         break;
 912
 913                 case ICMD_LADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
 914
 915                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 916                         x86_64_emit_lalu(cd, X86_64_ADD, src, iptr);
 917                         break;
 918
 919                 case ICMD_LADDCONST:  /* ..., value  ==> ..., value + constant        */
 920                                       /* val.l = constant                             */
 921
 922                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 923                         x86_64_emit_laluconst(cd, X86_64_ADD, src, iptr);
 924                         break;
 925
 926                 case ICMD_ISUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
 927
 928                         d = reg_of_var(rd, iptr->dst, REG_NULL);
 929                         if (iptr->dst->flags & INMEMORY) {
 930                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 931                                         if (src->prev->regoff == iptr->dst->regoff) {
 932                                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
 933                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 934
 935                                         } else {
 936                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 937                                                 x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 938                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 939                                         }
 940
 941                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 942                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
 943                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
 944                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 945
 946                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 947                                         if (src->prev->regoff == iptr->dst->regoff) {
 948                                                 x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 949
 950                                         } else {
 951                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 952                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 953                                                 x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
 954                                         }
 955
 956                                 } else {
 957                                         x86_64_movl_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
 958                                         x86_64_alul_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
 959                                 }
 960
 961                         } else {
 962                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 963                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 964                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 965
 966                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
 967                                         M_INTMOVE(src->prev->regoff, d);
 968                                         x86_64_alul_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
 969
 970                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
 971                                         /* workaround for reg alloc */
 972                                         if (src->regoff == iptr->dst->regoff) {
 973                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
 974                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 975                                                 M_INTMOVE(REG_ITMP1, d);
 976
 977                                         } else {
 978                                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
 979                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 980                                         }
 981
 982                                 } else {
 983                                         /* workaround for reg alloc */
 984                                         if (src->regoff == iptr->dst->regoff) {
 985                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
 986                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
 987                                                 M_INTMOVE(REG_ITMP1, d);
 988
 989                                         } else {
 990                                                 M_INTMOVE(src->prev->regoff, d);
 991                                                 x86_64_alul_reg_reg(cd, X86_64_SUB, src->regoff, d);
 992                                         }
 993                                 }
 994                         }
 995                         break;
 996
 997                 case ICMD_ISUBCONST:  /* ..., value  ==> ..., value + constant        */
 998                                       /* val.i = constant                             */
 999
1000                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1001                         x86_64_emit_ialuconst(cd, X86_64_SUB, src, iptr);
1002                         break;
1003
1004                 case ICMD_LSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1005
1006                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1007                         if (iptr->dst->flags & INMEMORY) {
1008                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1009                                         if (src->prev->regoff == iptr->dst->regoff) {
1010                                                 x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1011                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1012
1013                                         } else {
1014                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1015                                                 x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1016                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1017                                         }
1018
1019                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1020                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1021                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, REG_ITMP1);
1022                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1023
1024                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1025                                         if (src->prev->regoff == iptr->dst->regoff) {
1026                                                 x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1027
1028                                         } else {
1029                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1030                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1031                                                 x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1032                                         }
1033
1034                                 } else {
1035                                         x86_64_mov_reg_membase(cd, src->prev->regoff, REG_SP, iptr->dst->regoff * 8);
1036                                         x86_64_alu_reg_membase(cd, X86_64_SUB, src->regoff, REG_SP, iptr->dst->regoff * 8);
1037                                 }
1038
1039                         } else {
1040                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1041                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1042                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1043
1044                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1045                                         M_INTMOVE(src->prev->regoff, d);
1046                                         x86_64_alu_membase_reg(cd, X86_64_SUB, REG_SP, src->regoff * 8, d);
1047
1048                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1049                                         /* workaround for reg alloc */
1050                                         if (src->regoff == iptr->dst->regoff) {
1051                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1052                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1053                                                 M_INTMOVE(REG_ITMP1, d);
1054
1055                                         } else {
1056                                                 x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, d);
1057                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1058                                         }
1059
1060                                 } else {
1061                                         /* workaround for reg alloc */
1062                                         if (src->regoff == iptr->dst->regoff) {
1063                                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1064                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, REG_ITMP1);
1065                                                 M_INTMOVE(REG_ITMP1, d);
1066
1067                                         } else {
1068                                                 M_INTMOVE(src->prev->regoff, d);
1069                                                 x86_64_alu_reg_reg(cd, X86_64_SUB, src->regoff, d);
1070                                         }
1071                                 }
1072                         }
1073                         break;
1074
1075                 case ICMD_LSUBCONST:  /* ..., value  ==> ..., value - constant        */
1076                                       /* val.l = constant                             */
1077
1078                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1079                         x86_64_emit_laluconst(cd, X86_64_SUB, src, iptr);
1080                         break;
1081
1082                 case ICMD_IMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1083
1084                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1085                         if (iptr->dst->flags & INMEMORY) {
1086                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1087                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1088                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1089                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1090
1091                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1092                                         x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1093                                         x86_64_imull_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1094                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1095
1096                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1097                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1098                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1099                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1100
1101                                 } else {
1102                                         M_INTMOVE(src->prev->regoff, REG_ITMP1);
1103                                         x86_64_imull_reg_reg(cd, src->regoff, REG_ITMP1);
1104                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1105                                 }
1106
1107                         } else {
1108                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1109                                         x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1110                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1111
1112                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1113                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1114                                         x86_64_imull_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1115
1116                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1117                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1118                                         x86_64_imull_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1119
1120                                 } else {
1121                                         if (src->regoff == iptr->dst->regoff) {
1122                                                 x86_64_imull_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1123
1124                                         } else {
1125                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1126                                                 x86_64_imull_reg_reg(cd, src->regoff, iptr->dst->regoff);
1127                                         }
1128                                 }
1129                         }
1130                         break;
1131
1132                 case ICMD_IMULCONST:  /* ..., value  ==> ..., value * constant        */
1133                                       /* val.i = constant                             */
1134
1135                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1136                         if (iptr->dst->flags & INMEMORY) {
1137                                 if (src->flags & INMEMORY) {
1138                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, REG_ITMP1);
1139                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1140
1141                                 } else {
1142                                         x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, REG_ITMP1);
1143                                         x86_64_movl_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1144                                 }
1145
1146                         } else {
1147                                 if (src->flags & INMEMORY) {
1148                                         x86_64_imull_imm_membase_reg(cd, iptr->val.i, REG_SP, src->regoff * 8, iptr->dst->regoff);
1149
1150                                 } else {
1151                                         if (iptr->val.i == 2) {
1152                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1153                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1154
1155                                         } else {
1156                                                 x86_64_imull_imm_reg_reg(cd, iptr->val.i, src->regoff, iptr->dst->regoff);    /* 3 cycles */
1157                                         }
1158                                 }
1159                         }
1160                         break;
1161
1162                 case ICMD_LMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1163
1164                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1165                         if (iptr->dst->flags & INMEMORY) {
1166                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1167                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1168                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1169                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1170
1171                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1172                                         x86_64_mov_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1173                                         x86_64_imul_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1174                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1175
1176                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1177                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, REG_ITMP1);
1178                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1179                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1180
1181                                 } else {
1182                                         x86_64_mov_reg_reg(cd, src->prev->regoff, REG_ITMP1);
1183                                         x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1184                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1185                                 }
1186
1187                         } else {
1188                                 if ((src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1189                                         x86_64_mov_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1190                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1191
1192                                 } else if ((src->flags & INMEMORY) && !(src->prev->flags & INMEMORY)) {
1193                                         M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1194                                         x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1195
1196                                 } else if (!(src->flags & INMEMORY) && (src->prev->flags & INMEMORY)) {
1197                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1198                                         x86_64_imul_membase_reg(cd, REG_SP, src->prev->regoff * 8, iptr->dst->regoff);
1199
1200                                 } else {
1201                                         if (src->regoff == iptr->dst->regoff) {
1202                                                 x86_64_imul_reg_reg(cd, src->prev->regoff, iptr->dst->regoff);
1203
1204                                         } else {
1205                                                 M_INTMOVE(src->prev->regoff, iptr->dst->regoff);
1206                                                 x86_64_imul_reg_reg(cd, src->regoff, iptr->dst->regoff);
1207                                         }
1208                                 }
1209                         }
1210                         break;
1211
1212                 case ICMD_LMULCONST:  /* ..., value  ==> ..., value * constant        */
1213                                       /* val.l = constant                             */
1214
1215                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1216                         if (iptr->dst->flags & INMEMORY) {
1217                                 if (src->flags & INMEMORY) {
1218                                         if (IS_IMM32(iptr->val.l)) {
1219                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, REG_ITMP1);
1220
1221                                         } else {
1222                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1223                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP1);
1224                                         }
1225                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1226
1227                                 } else {
1228                                         if (IS_IMM32(iptr->val.l)) {
1229                                                 x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, REG_ITMP1);
1230
1231                                         } else {
1232                                                 x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1233                                                 x86_64_imul_reg_reg(cd, src->regoff, REG_ITMP1);
1234                                         }
1235                                         x86_64_mov_reg_membase(cd, REG_ITMP1, REG_SP, iptr->dst->regoff * 8);
1236                                 }
1237
1238                         } else {
1239                                 if (src->flags & INMEMORY) {
1240                                         if (IS_IMM32(iptr->val.l)) {
1241                                                 x86_64_imul_imm_membase_reg(cd, iptr->val.l, REG_SP, src->regoff * 8, iptr->dst->regoff);
1242
1243                                         } else {
1244                                                 x86_64_mov_imm_reg(cd, iptr->val.l, iptr->dst->regoff);
1245                                                 x86_64_imul_membase_reg(cd, REG_SP, src->regoff * 8, iptr->dst->regoff);
1246                                         }
1247
1248                                 } else {
1249                                         /* should match in many cases */
1250                                         if (iptr->val.l == 2) {
1251                                                 M_INTMOVE(src->regoff, iptr->dst->regoff);
1252                                                 x86_64_alul_reg_reg(cd, X86_64_ADD, iptr->dst->regoff, iptr->dst->regoff);
1253
1254                                         } else {
1255                                                 if (IS_IMM32(iptr->val.l)) {
1256                                                         x86_64_imul_imm_reg_reg(cd, iptr->val.l, src->regoff, iptr->dst->regoff);    /* 4 cycles */
1257
1258                                                 } else {
1259                                                         x86_64_mov_imm_reg(cd, iptr->val.l, REG_ITMP1);
1260                                                         M_INTMOVE(src->regoff, iptr->dst->regoff);
1261                                                         x86_64_imul_reg_reg(cd, REG_ITMP1, iptr->dst->regoff);
1262                                                 }
1263                                         }
1264                                 }
1265                         }
1266                         break;
1267
1268                 case ICMD_IDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1269
1270                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1271                 if (src->prev->flags & INMEMORY) {
1272                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1273
1274                         } else {
1275                                 M_INTMOVE(src->prev->regoff, RAX);
1276                         }
1277
1278                         if (src->flags & INMEMORY) {
1279                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1280
1281                         } else {
1282                                 M_INTMOVE(src->regoff, REG_ITMP3);
1283                         }
1284                         gen_div_check(src);
1285
1286                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1287                         x86_64_jcc(cd, X86_64_CC_NE, 4 + 6);
1288                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1289                         x86_64_jcc(cd, X86_64_CC_E, 3 + 1 + 3);                /* 6 bytes */
1290
1291                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1292                         x86_64_cltd(cd);
1293                         x86_64_idivl_reg(cd, REG_ITMP3);
1294
1295                         if (iptr->dst->flags & INMEMORY) {
1296                                 x86_64_mov_reg_membase(cd, RAX, REG_SP, iptr->dst->regoff * 8);
1297                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1298
1299                         } else {
1300                                 M_INTMOVE(RAX, iptr->dst->regoff);
1301
1302                                 if (iptr->dst->regoff != RDX) {
1303                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1304                                 }
1305                         }
1306                         break;
1307
1308                 case ICMD_IREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1309                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1310                         if (src->prev->flags & INMEMORY) {
1311                                 x86_64_movl_membase_reg(cd, REG_SP, src->prev->regoff * 8, RAX);
1312
1313                         } else {
1314                                 M_INTMOVE(src->prev->regoff, RAX);
1315                         }
1316
1317                         if (src->flags & INMEMORY) {
1318                                 x86_64_movl_membase_reg(cd, REG_SP, src->regoff * 8, REG_ITMP3);
1319
1320                         } else {
1321                                 M_INTMOVE(src->regoff, REG_ITMP3);
1322                         }
1323                         gen_div_check(src);
1324
1325                         x86_64_mov_reg_reg(cd, RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1326
1327                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, RAX); /* check as described in jvm spec */
1328                         x86_64_jcc(cd, X86_64_CC_NE, 2 + 4 + 6);
1329
1330
1331                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1332                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP3);    /* 4 bytes */
1333                         x86_64_jcc(cd, X86_64_CC_E, 1 + 3);                    /* 6 bytes */
1334
1335                         x86_64_cltd(cd);
1336                         x86_64_idivl_reg(cd, REG_ITMP3);
1337
1338                         if (iptr->dst->flags & INMEMORY) {
1339                                 x86_64_mov_reg_membase(cd, RDX, REG_SP, iptr->dst->regoff * 8);
1340                                 x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);       /* restore %rdx */
1341
1342                         } else {
1343                                 M_INTMOVE(RDX, iptr->dst->regoff);
1344
1345                                 if (iptr->dst->regoff != RDX) {
1346                                         x86_64_mov_reg_reg(cd, REG_ITMP2, RDX);   /* restore %rdx */
1347                                 }
1348                         }
1349                         break;
1350
1351                 case ICMD_IDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1352                                       /* val.i = constant                             */
1353
1354                         var_to_reg_int(s1, src, REG_ITMP1);
1355                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1356                         M_INTMOVE(s1, REG_ITMP1);
1357                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1358                         x86_64_leal_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1359                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1360                         x86_64_shiftl_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1361                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1362                         store_reg_to_var_int(iptr->dst, d);
1363                         break;
1364
1365                 case ICMD_IREMPOW2:   /* ..., value  ==> ..., value % constant        */
1366                                       /* val.i = constant                             */
1367
1368                         var_to_reg_int(s1, src, REG_ITMP1);
1369                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1370                         M_INTMOVE(s1, REG_ITMP1);
1371                         x86_64_alul_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1372                         x86_64_leal_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1373                         x86_64_cmovccl_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1374                         x86_64_alul_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1375                         x86_64_alul_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1376                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1377                         store_reg_to_var_int(iptr->dst, d);
1378                         break;
1379
1380
1381                 case ICMD_LDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1382
1383                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1384
1385                 if (src->prev->flags & INMEMORY) {
1386                                 M_LLD(RAX, REG_SP, src->prev->regoff * 8);
1387
1388                         } else {
1389                                 M_INTMOVE(src->prev->regoff, RAX);
1390                         }
1391
1392                         if (src->flags & INMEMORY) {
1393                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1394
1395                         } else {
1396                                 M_INTMOVE(src->regoff, REG_ITMP3);
1397                         }
1398                         gen_div_check(src);
1399
1400                         /* check as described in jvm spec */
1401                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1402                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, RAX);
1403                         M_BNE(4 + 6);
1404                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1405                         M_BEQ(3 + 2 + 3);                                      /* 6 bytes */
1406
1407                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1408                         x86_64_cqto(cd);
1409                         x86_64_idiv_reg(cd, REG_ITMP3);
1410
1411                         if (iptr->dst->flags & INMEMORY) {
1412                                 M_LST(RAX, REG_SP, iptr->dst->regoff * 8);
1413                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1414
1415                         } else {
1416                                 M_INTMOVE(RAX, iptr->dst->regoff);
1417
1418                                 if (iptr->dst->regoff != RDX) {
1419                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1420                                 }
1421                         }
1422                         break;
1423
1424                 case ICMD_LREM:       /* ..., val1, val2  ==> ..., val1 % val2        */
1425
1426                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1427                         if (src->prev->flags & INMEMORY) {
1428                                 M_LLD(REG_ITMP1, REG_SP, src->prev->regoff * 8);
1429
1430                         } else {
1431                                 M_INTMOVE(src->prev->regoff, REG_ITMP1);
1432                         }
1433
1434                         if (src->flags & INMEMORY) {
1435                                 M_LLD(REG_ITMP3, REG_SP, src->regoff * 8);
1436
1437                         } else {
1438                                 M_INTMOVE(src->regoff, REG_ITMP3);
1439                         }
1440                         gen_div_check(src);
1441
1442                         M_MOV(RDX, REG_ITMP2); /* save %rdx, cause it's an argument register */
1443
1444                         /* check as described in jvm spec */
1445                         disp = dseg_adds8(cd, 0x8000000000000000LL);
1446                         M_CMP_MEMBASE(RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP1);
1447                         M_BNE(3 + 4 + 6);
1448
1449 #if 0
1450                         x86_64_alul_reg_reg(cd, X86_64_XOR, RDX, RDX);         /* 2 bytes */
1451 #endif
1452                         M_XOR(RDX, RDX);                                       /* 3 bytes */
1453                         M_CMP_IMM(-1, REG_ITMP3);                              /* 4 bytes */
1454                         M_BEQ(2 + 3);                                          /* 6 bytes */
1455
1456                         x86_64_cqto(cd);
1457                         x86_64_idiv_reg(cd, REG_ITMP3);
1458
1459                         if (iptr->dst->flags & INMEMORY) {
1460                                 M_LST(RDX, REG_SP, iptr->dst->regoff * 8);
1461                                 M_MOV(REG_ITMP2, RDX);                        /* restore %rdx */
1462
1463                         } else {
1464                                 M_INTMOVE(RDX, iptr->dst->regoff);
1465
1466                                 if (iptr->dst->regoff != RDX) {
1467                                         M_MOV(REG_ITMP2, RDX);                    /* restore %rdx */
1468                                 }
1469                         }
1470                         break;
1471
1472                 case ICMD_LDIVPOW2:   /* ..., value  ==> ..., value >> constant       */
1473                                       /* val.i = constant                             */
1474
1475                         var_to_reg_int(s1, src, REG_ITMP1);
1476                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1477                         M_INTMOVE(s1, REG_ITMP1);
1478                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1479                         x86_64_lea_membase_reg(cd, REG_ITMP1, (1 << iptr->val.i) - 1, REG_ITMP2);
1480                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_LE, REG_ITMP2, REG_ITMP1);
1481                         x86_64_shift_imm_reg(cd, X86_64_SAR, iptr->val.i, REG_ITMP1);
1482                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1483                         store_reg_to_var_int(iptr->dst, d);
1484                         break;
1485
1486                 case ICMD_LREMPOW2:   /* ..., value  ==> ..., value % constant        */
1487                                       /* val.l = constant                             */
1488
1489                         var_to_reg_int(s1, src, REG_ITMP1);
1490                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1491                         M_INTMOVE(s1, REG_ITMP1);
1492                         x86_64_alu_imm_reg(cd, X86_64_CMP, -1, REG_ITMP1);
1493                         x86_64_lea_membase_reg(cd, REG_ITMP1, iptr->val.i, REG_ITMP2);
1494                         x86_64_cmovcc_reg_reg(cd, X86_64_CC_G, REG_ITMP1, REG_ITMP2);
1495                         x86_64_alu_imm_reg(cd, X86_64_AND, -1 - (iptr->val.i), REG_ITMP2);
1496                         x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
1497                         x86_64_mov_reg_reg(cd, REG_ITMP1, d);
1498                         store_reg_to_var_int(iptr->dst, d);
1499                         break;
1500
1501                 case ICMD_ISHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1502
1503                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1504                         x86_64_emit_ishift(cd, X86_64_SHL, src, iptr);
1505                         break;
1506
1507                 case ICMD_ISHLCONST:  /* ..., value  ==> ..., value << constant       */
1508                                       /* val.i = constant                             */
1509
1510                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1511                         x86_64_emit_ishiftconst(cd, X86_64_SHL, src, iptr);
1512                         break;
1513
1514                 case ICMD_ISHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1515
1516                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1517                         x86_64_emit_ishift(cd, X86_64_SAR, src, iptr);
1518                         break;
1519
1520                 case ICMD_ISHRCONST:  /* ..., value  ==> ..., value >> constant       */
1521                                       /* val.i = constant                             */
1522
1523                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1524                         x86_64_emit_ishiftconst(cd, X86_64_SAR, src, iptr);
1525                         break;
1526
1527                 case ICMD_IUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1528
1529                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1530                         x86_64_emit_ishift(cd, X86_64_SHR, src, iptr);
1531                         break;
1532
1533                 case ICMD_IUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1534                                       /* val.i = constant                             */
1535
1536                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1537                         x86_64_emit_ishiftconst(cd, X86_64_SHR, src, iptr);
1538                         break;
1539
1540                 case ICMD_LSHL:       /* ..., val1, val2  ==> ..., val1 << val2       */
1541
1542                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1543                         x86_64_emit_lshift(cd, X86_64_SHL, src, iptr);
1544                         break;
1545
1546         case ICMD_LSHLCONST:  /* ..., value  ==> ..., value << constant       */
1547                                           /* val.i = constant                             */
1548
1549                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1550                         x86_64_emit_lshiftconst(cd, X86_64_SHL, src, iptr);
1551                         break;
1552
1553                 case ICMD_LSHR:       /* ..., val1, val2  ==> ..., val1 >> val2       */
1554
1555                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1556                         x86_64_emit_lshift(cd, X86_64_SAR, src, iptr);
1557                         break;
1558
1559                 case ICMD_LSHRCONST:  /* ..., value  ==> ..., value >> constant       */
1560                                       /* val.i = constant                             */
1561
1562                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1563                         x86_64_emit_lshiftconst(cd, X86_64_SAR, src, iptr);
1564                         break;
1565
1566                 case ICMD_LUSHR:      /* ..., val1, val2  ==> ..., val1 >>> val2      */
1567
1568                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1569                         x86_64_emit_lshift(cd, X86_64_SHR, src, iptr);
1570                         break;
1571
1572                 case ICMD_LUSHRCONST: /* ..., value  ==> ..., value >>> constant      */
1573                                       /* val.l = constant                             */
1574
1575                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1576                         x86_64_emit_lshiftconst(cd, X86_64_SHR, src, iptr);
1577                         break;
1578
1579                 case ICMD_IAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1580
1581                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1582                         x86_64_emit_ialu(cd, X86_64_AND, src, iptr);
1583                         break;
1584
1585                 case ICMD_IANDCONST:  /* ..., value  ==> ..., value & constant        */
1586                                       /* val.i = constant                             */
1587
1588                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1589                         x86_64_emit_ialuconst(cd, X86_64_AND, src, iptr);
1590                         break;
1591
1592                 case ICMD_LAND:       /* ..., val1, val2  ==> ..., val1 & val2        */
1593
1594                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1595                         x86_64_emit_lalu(cd, X86_64_AND, src, iptr);
1596                         break;
1597
1598                 case ICMD_LANDCONST:  /* ..., value  ==> ..., value & constant        */
1599                                       /* val.l = constant                             */
1600
1601                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1602                         x86_64_emit_laluconst(cd, X86_64_AND, src, iptr);
1603                         break;
1604
1605                 case ICMD_IOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1606
1607                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1608                         x86_64_emit_ialu(cd, X86_64_OR, src, iptr);
1609                         break;
1610
1611                 case ICMD_IORCONST:   /* ..., value  ==> ..., value | constant        */
1612                                       /* val.i = constant                             */
1613
1614                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1615                         x86_64_emit_ialuconst(cd, X86_64_OR, src, iptr);
1616                         break;
1617
1618                 case ICMD_LOR:        /* ..., val1, val2  ==> ..., val1 | val2        */
1619
1620                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1621                         x86_64_emit_lalu(cd, X86_64_OR, src, iptr);
1622                         break;
1623
1624                 case ICMD_LORCONST:   /* ..., value  ==> ..., value | constant        */
1625                                       /* val.l = constant                             */
1626
1627                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1628                         x86_64_emit_laluconst(cd, X86_64_OR, src, iptr);
1629                         break;
1630
1631                 case ICMD_IXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1632
1633                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1634                         x86_64_emit_ialu(cd, X86_64_XOR, src, iptr);
1635                         break;
1636
1637                 case ICMD_IXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1638                                       /* val.i = constant                             */
1639
1640                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1641                         x86_64_emit_ialuconst(cd, X86_64_XOR, src, iptr);
1642                         break;
1643
1644                 case ICMD_LXOR:       /* ..., val1, val2  ==> ..., val1 ^ val2        */
1645
1646                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1647                         x86_64_emit_lalu(cd, X86_64_XOR, src, iptr);
1648                         break;
1649
1650                 case ICMD_LXORCONST:  /* ..., value  ==> ..., value ^ constant        */
1651                                       /* val.l = constant                             */
1652
1653                         d = reg_of_var(rd, iptr->dst, REG_NULL);
1654                         x86_64_emit_laluconst(cd, X86_64_XOR, src, iptr);
1655                         break;
1656
1657
1658                 case ICMD_IINC:       /* ..., value  ==> ..., value + constant        */
1659                                       /* op1 = variable, val.i = constant             */
1660
1661                         /* using inc and dec is definitely faster than add -- tested      */
1662                         /* with sieve                                                     */
1663
1664                         var = &(rd->locals[iptr->op1][TYPE_INT]);
1665                         d = var->regoff;
1666                         if (var->flags & INMEMORY) {
1667                                 if (iptr->val.i == 1) {
1668                                         x86_64_incl_membase(cd, REG_SP, d * 8);
1669
1670                                 } else if (iptr->val.i == -1) {
1671                                         x86_64_decl_membase(cd, REG_SP, d * 8);
1672
1673                                 } else {
1674                                         x86_64_alul_imm_membase(cd, X86_64_ADD, iptr->val.i, REG_SP, d * 8);
1675                                 }
1676
1677                         } else {
1678                                 if (iptr->val.i == 1) {
1679                                         x86_64_incl_reg(cd, d);
1680
1681                                 } else if (iptr->val.i == -1) {
1682                                         x86_64_decl_reg(cd, d);
1683
1684                                 } else {
1685                                         x86_64_alul_imm_reg(cd, X86_64_ADD, iptr->val.i, d);
1686                                 }
1687                         }
1688                         break;
1689
1690
1691                 /* floating operations ************************************************/
1692
1693                 case ICMD_FNEG:       /* ..., value  ==> ..., - value                 */
1694
1695                         var_to_reg_flt(s1, src, REG_FTMP1);
1696                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1697                         disp = dseg_adds4(cd, 0x80000000);
1698                         M_FLTMOVE(s1, d);
1699                         x86_64_movss_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1700                         x86_64_xorps_reg_reg(cd, REG_FTMP2, d);
1701                         store_reg_to_var_flt(iptr->dst, d);
1702                         break;
1703
1704                 case ICMD_DNEG:       /* ..., value  ==> ..., - value                 */
1705
1706                         var_to_reg_flt(s1, src, REG_FTMP1);
1707                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1708                         disp = dseg_adds8(cd, 0x8000000000000000);
1709                         M_FLTMOVE(s1, d);
1710                         x86_64_movd_membase_reg(cd, RIP, -(((s8) cd->mcodeptr + 9) - (s8) cd->mcodebase) + disp, REG_FTMP2);
1711                         x86_64_xorpd_reg_reg(cd, REG_FTMP2, d);
1712                         store_reg_to_var_flt(iptr->dst, d);
1713                         break;
1714
1715                 case ICMD_FADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1716
1717                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1718                         var_to_reg_flt(s2, src, REG_FTMP2);
1719                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1720                         if (s1 == d) {
1721                                 x86_64_addss_reg_reg(cd, s2, d);
1722                         } else if (s2 == d) {
1723                                 x86_64_addss_reg_reg(cd, s1, d);
1724                         } else {
1725                                 M_FLTMOVE(s1, d);
1726                                 x86_64_addss_reg_reg(cd, s2, d);
1727                         }
1728                         store_reg_to_var_flt(iptr->dst, d);
1729                         break;
1730
1731                 case ICMD_DADD:       /* ..., val1, val2  ==> ..., val1 + val2        */
1732
1733                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1734                         var_to_reg_flt(s2, src, REG_FTMP2);
1735                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1736                         if (s1 == d) {
1737                                 x86_64_addsd_reg_reg(cd, s2, d);
1738                         } else if (s2 == d) {
1739                                 x86_64_addsd_reg_reg(cd, s1, d);
1740                         } else {
1741                                 M_FLTMOVE(s1, d);
1742                                 x86_64_addsd_reg_reg(cd, s2, d);
1743                         }
1744                         store_reg_to_var_flt(iptr->dst, d);
1745                         break;
1746
1747                 case ICMD_FSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1748
1749                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1750                         var_to_reg_flt(s2, src, REG_FTMP2);
1751                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1752                         if (s2 == d) {
1753                                 M_FLTMOVE(s2, REG_FTMP2);
1754                                 s2 = REG_FTMP2;
1755                         }
1756                         M_FLTMOVE(s1, d);
1757                         x86_64_subss_reg_reg(cd, s2, d);
1758                         store_reg_to_var_flt(iptr->dst, d);
1759                         break;
1760
1761                 case ICMD_DSUB:       /* ..., val1, val2  ==> ..., val1 - val2        */
1762
1763                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1764                         var_to_reg_flt(s2, src, REG_FTMP2);
1765                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1766                         if (s2 == d) {
1767                                 M_FLTMOVE(s2, REG_FTMP2);
1768                                 s2 = REG_FTMP2;
1769                         }
1770                         M_FLTMOVE(s1, d);
1771                         x86_64_subsd_reg_reg(cd, s2, d);
1772                         store_reg_to_var_flt(iptr->dst, d);
1773                         break;
1774
1775                 case ICMD_FMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1776
1777                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1778                         var_to_reg_flt(s2, src, REG_FTMP2);
1779                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1780                         if (s1 == d) {
1781                                 x86_64_mulss_reg_reg(cd, s2, d);
1782                         } else if (s2 == d) {
1783                                 x86_64_mulss_reg_reg(cd, s1, d);
1784                         } else {
1785                                 M_FLTMOVE(s1, d);
1786                                 x86_64_mulss_reg_reg(cd, s2, d);
1787                         }
1788                         store_reg_to_var_flt(iptr->dst, d);
1789                         break;
1790
1791                 case ICMD_DMUL:       /* ..., val1, val2  ==> ..., val1 * val2        */
1792
1793                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1794                         var_to_reg_flt(s2, src, REG_FTMP2);
1795                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1796                         if (s1 == d) {
1797                                 x86_64_mulsd_reg_reg(cd, s2, d);
1798                         } else if (s2 == d) {
1799                                 x86_64_mulsd_reg_reg(cd, s1, d);
1800                         } else {
1801                                 M_FLTMOVE(s1, d);
1802                                 x86_64_mulsd_reg_reg(cd, s2, d);
1803                         }
1804                         store_reg_to_var_flt(iptr->dst, d);
1805                         break;
1806
1807                 case ICMD_FDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1808
1809                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1810                         var_to_reg_flt(s2, src, REG_FTMP2);
1811                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1812                         if (s2 == d) {
1813                                 M_FLTMOVE(s2, REG_FTMP2);
1814                                 s2 = REG_FTMP2;
1815                         }
1816                         M_FLTMOVE(s1, d);
1817                         x86_64_divss_reg_reg(cd, s2, d);
1818                         store_reg_to_var_flt(iptr->dst, d);
1819                         break;
1820
1821                 case ICMD_DDIV:       /* ..., val1, val2  ==> ..., val1 / val2        */
1822
1823                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1824                         var_to_reg_flt(s2, src, REG_FTMP2);
1825                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1826                         if (s2 == d) {
1827                                 M_FLTMOVE(s2, REG_FTMP2);
1828                                 s2 = REG_FTMP2;
1829                         }
1830                         M_FLTMOVE(s1, d);
1831                         x86_64_divsd_reg_reg(cd, s2, d);
1832                         store_reg_to_var_flt(iptr->dst, d);
1833                         break;
1834
1835                 case ICMD_I2F:       /* ..., value  ==> ..., (float) value            */
1836
1837                         var_to_reg_int(s1, src, REG_ITMP1);
1838                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1839                         x86_64_cvtsi2ss_reg_reg(cd, s1, d);
1840                         store_reg_to_var_flt(iptr->dst, d);
1841                         break;
1842
1843                 case ICMD_I2D:       /* ..., value  ==> ..., (double) value           */
1844
1845                         var_to_reg_int(s1, src, REG_ITMP1);
1846                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1847                         x86_64_cvtsi2sd_reg_reg(cd, s1, d);
1848                         store_reg_to_var_flt(iptr->dst, d);
1849                         break;
1850
1851                 case ICMD_L2F:       /* ..., value  ==> ..., (float) value            */
1852
1853                         var_to_reg_int(s1, src, REG_ITMP1);
1854                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1855                         x86_64_cvtsi2ssq_reg_reg(cd, s1, d);
1856                         store_reg_to_var_flt(iptr->dst, d);
1857                         break;
1858
1859                 case ICMD_L2D:       /* ..., value  ==> ..., (double) value           */
1860
1861                         var_to_reg_int(s1, src, REG_ITMP1);
1862                         d = reg_of_var(rd, iptr->dst, REG_FTMP1);
1863                         x86_64_cvtsi2sdq_reg_reg(cd, s1, d);
1864                         store_reg_to_var_flt(iptr->dst, d);
1865                         break;
1866
1867                 case ICMD_F2I:       /* ..., value  ==> ..., (int) value              */
1868
1869                         var_to_reg_flt(s1, src, REG_FTMP1);
1870                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1871                         x86_64_cvttss2si_reg_reg(cd, s1, d);
1872                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1873                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1874                         x86_64_jcc(cd, X86_64_CC_NE, a);
1875                         M_FLTMOVE(s1, REG_FTMP1);
1876                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2i, REG_ITMP2);
1877                         x86_64_call_reg(cd, REG_ITMP2);
1878                         M_INTMOVE(REG_RESULT, d);
1879                         store_reg_to_var_int(iptr->dst, d);
1880                         break;
1881
1882                 case ICMD_D2I:       /* ..., value  ==> ..., (int) value              */
1883
1884                         var_to_reg_flt(s1, src, REG_FTMP1);
1885                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1886                         x86_64_cvttsd2si_reg_reg(cd, s1, d);
1887                         x86_64_alul_imm_reg(cd, X86_64_CMP, 0x80000000, d);    /* corner cases */
1888                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1889                         x86_64_jcc(cd, X86_64_CC_NE, a);
1890                         M_FLTMOVE(s1, REG_FTMP1);
1891                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2i, REG_ITMP2);
1892                         x86_64_call_reg(cd, REG_ITMP2);
1893                         M_INTMOVE(REG_RESULT, d);
1894                         store_reg_to_var_int(iptr->dst, d);
1895                         break;
1896
1897                 case ICMD_F2L:       /* ..., value  ==> ..., (long) value             */
1898
1899                         var_to_reg_flt(s1, src, REG_FTMP1);
1900                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1901                         x86_64_cvttss2siq_reg_reg(cd, s1, d);
1902                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1903                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1904                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1905                         x86_64_jcc(cd, X86_64_CC_NE, a);
1906                         M_FLTMOVE(s1, REG_FTMP1);
1907                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_f2l, REG_ITMP2);
1908                         x86_64_call_reg(cd, REG_ITMP2);
1909                         M_INTMOVE(REG_RESULT, d);
1910                         store_reg_to_var_int(iptr->dst, d);
1911                         break;
1912
1913                 case ICMD_D2L:       /* ..., value  ==> ..., (long) value             */
1914
1915                         var_to_reg_flt(s1, src, REG_FTMP1);
1916                         d = reg_of_var(rd, iptr->dst, REG_ITMP1);
1917                         x86_64_cvttsd2siq_reg_reg(cd, s1, d);
1918                         x86_64_mov_imm_reg(cd, 0x8000000000000000, REG_ITMP2);
1919                         x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP2, d);     /* corner cases */
1920                         a = ((s1 == REG_FTMP1) ? 0 : 5) + 10 + 3 + ((REG_RESULT == d) ? 0 : 3);
1921                         x86_64_jcc(cd, X86_64_CC_NE, a);
1922                         M_FLTMOVE(s1, REG_FTMP1);
1923                         x86_64_mov_imm_reg(cd, (ptrint) asm_builtin_d2l, REG_ITMP2);
1924                         x86_64_call_reg(cd, REG_ITMP2);
1925                         M_INTMOVE(REG_RESULT, d);
1926                         store_reg_to_var_int(iptr->dst, d);
1927                         break;
1928
1929                 case ICMD_F2D:       /* ..., value  ==> ..., (double) value           */
1930
1931                         var_to_reg_flt(s1, src, REG_FTMP1);
1932                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1933                         x86_64_cvtss2sd_reg_reg(cd, s1, d);
1934                         store_reg_to_var_flt(iptr->dst, d);
1935                         break;
1936
1937                 case ICMD_D2F:       /* ..., value  ==> ..., (float) value            */
1938
1939                         var_to_reg_flt(s1, src, REG_FTMP1);
1940                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
1941                         x86_64_cvtsd2ss_reg_reg(cd, s1, d);
1942                         store_reg_to_var_flt(iptr->dst, d);
1943                         break;
1944
1945                 case ICMD_FCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1946                                           /* == => 0, < => 1, > => -1 */
1947
1948                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1949                         var_to_reg_flt(s2, src, REG_FTMP2);
1950                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1951                         M_CLR(d);
1952                         M_MOV_IMM(1, REG_ITMP1);
1953                         M_MOV_IMM(-1, REG_ITMP2);
1954                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1955                         M_CMOVB(REG_ITMP1, d);
1956                         M_CMOVA(REG_ITMP2, d);
1957                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1958                         store_reg_to_var_int(iptr->dst, d);
1959                         break;
1960
1961                 case ICMD_FCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1962                                           /* == => 0, < => 1, > => -1 */
1963
1964                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1965                         var_to_reg_flt(s2, src, REG_FTMP2);
1966                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1967                         M_CLR(d);
1968                         M_MOV_IMM(1, REG_ITMP1);
1969                         M_MOV_IMM(-1, REG_ITMP2);
1970                         x86_64_ucomiss_reg_reg(cd, s1, s2);
1971                         M_CMOVB(REG_ITMP1, d);
1972                         M_CMOVA(REG_ITMP2, d);
1973                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
1974                         store_reg_to_var_int(iptr->dst, d);
1975                         break;
1976
1977                 case ICMD_DCMPL:      /* ..., val1, val2  ==> ..., val1 fcmpl val2    */
1978                                           /* == => 0, < => 1, > => -1 */
1979
1980                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1981                         var_to_reg_flt(s2, src, REG_FTMP2);
1982                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1983                         M_CLR(d);
1984                         M_MOV_IMM(1, REG_ITMP1);
1985                         M_MOV_IMM(-1, REG_ITMP2);
1986                         x86_64_ucomisd_reg_reg(cd, s1, s2);
1987                         M_CMOVB(REG_ITMP1, d);
1988                         M_CMOVA(REG_ITMP2, d);
1989                         M_CMOVP(REG_ITMP2, d);                   /* treat unordered as GT */
1990                         store_reg_to_var_int(iptr->dst, d);
1991                         break;
1992
1993                 case ICMD_DCMPG:      /* ..., val1, val2  ==> ..., val1 fcmpg val2    */
1994                                           /* == => 0, < => 1, > => -1 */
1995
1996                         var_to_reg_flt(s1, src->prev, REG_FTMP1);
1997                         var_to_reg_flt(s2, src, REG_FTMP2);
1998                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
1999                         M_CLR(d);
2000                         M_MOV_IMM(1, REG_ITMP1);
2001                         M_MOV_IMM(-1, REG_ITMP2);
2002                         x86_64_ucomisd_reg_reg(cd, s1, s2);
2003                         M_CMOVB(REG_ITMP1, d);
2004                         M_CMOVA(REG_ITMP2, d);
2005                         M_CMOVP(REG_ITMP1, d);                   /* treat unordered as LT */
2006                         store_reg_to_var_int(iptr->dst, d);
2007                         break;
2008
2009
2010                 /* memory operations **************************************************/
2011
2012                 case ICMD_ARRAYLENGTH: /* ..., arrayref  ==> ..., (int) length        */
2013
2014                         var_to_reg_int(s1, src, REG_ITMP1);
2015                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2016                         gen_nullptr_check(s1);
2017                         M_ILD(d, s1, OFFSET(java_arrayheader, size));
2018                         store_reg_to_var_int(iptr->dst, d);
2019                         break;
2020
2021                 case ICMD_BALOAD:     /* ..., arrayref, index  ==> ..., value         */
2022
2023                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2024                         var_to_reg_int(s2, src, REG_ITMP2);
2025                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2026                         if (iptr->op1 == 0) {
2027                                 gen_nullptr_check(s1);
2028                                 gen_bound_check;
2029                         }
2030                         x86_64_movsbq_memindex_reg(cd, OFFSET(java_bytearray, data[0]), s1, s2, 0, d);
2031                         store_reg_to_var_int(iptr->dst, d);
2032                         break;
2033
2034                 case ICMD_CALOAD:     /* ..., arrayref, index  ==> ..., value         */
2035
2036                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2037                         var_to_reg_int(s2, src, REG_ITMP2);
2038                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2039                         if (iptr->op1 == 0) {
2040                                 gen_nullptr_check(s1);
2041                                 gen_bound_check;
2042                         }
2043                         x86_64_movzwq_memindex_reg(cd, OFFSET(java_chararray, data[0]), s1, s2, 1, d);
2044                         store_reg_to_var_int(iptr->dst, d);
2045                         break;
2046
2047                 case ICMD_SALOAD:     /* ..., arrayref, index  ==> ..., value         */
2048
2049                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2050                         var_to_reg_int(s2, src, REG_ITMP2);
2051                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2052                         if (iptr->op1 == 0) {
2053                                 gen_nullptr_check(s1);
2054                                 gen_bound_check;
2055                         }
2056                         x86_64_movswq_memindex_reg(cd, OFFSET(java_shortarray, data[0]), s1, s2, 1, d);
2057                         store_reg_to_var_int(iptr->dst, d);
2058                         break;
2059
2060                 case ICMD_IALOAD:     /* ..., arrayref, index  ==> ..., value         */
2061
2062                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2063                         var_to_reg_int(s2, src, REG_ITMP2);
2064                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2065                         if (iptr->op1 == 0) {
2066                                 gen_nullptr_check(s1);
2067                                 gen_bound_check;
2068                         }
2069                         x86_64_movl_memindex_reg(cd, OFFSET(java_intarray, data[0]), s1, s2, 2, d);
2070                         store_reg_to_var_int(iptr->dst, d);
2071                         break;
2072
2073                 case ICMD_LALOAD:     /* ..., arrayref, index  ==> ..., value         */
2074
2075                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2076                         var_to_reg_int(s2, src, REG_ITMP2);
2077                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2078                         if (iptr->op1 == 0) {
2079                                 gen_nullptr_check(s1);
2080                                 gen_bound_check;
2081                         }
2082                         x86_64_mov_memindex_reg(cd, OFFSET(java_longarray, data[0]), s1, s2, 3, d);
2083                         store_reg_to_var_int(iptr->dst, d);
2084                         break;
2085
2086                 case ICMD_FALOAD:     /* ..., arrayref, index  ==> ..., value         */
2087
2088                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2089                         var_to_reg_int(s2, src, REG_ITMP2);
2090                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2091                         if (iptr->op1 == 0) {
2092                                 gen_nullptr_check(s1);
2093                                 gen_bound_check;
2094                         }
2095                         x86_64_movss_memindex_reg(cd, OFFSET(java_floatarray, data[0]), s1, s2, 2, d);
2096                         store_reg_to_var_flt(iptr->dst, d);
2097                         break;
2098
2099                 case ICMD_DALOAD:     /* ..., arrayref, index  ==> ..., value         */
2100
2101                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2102                         var_to_reg_int(s2, src, REG_ITMP2);
2103                         d = reg_of_var(rd, iptr->dst, REG_FTMP3);
2104                         if (iptr->op1 == 0) {
2105                                 gen_nullptr_check(s1);
2106                                 gen_bound_check;
2107                         }
2108                         x86_64_movsd_memindex_reg(cd, OFFSET(java_doublearray, data[0]), s1, s2, 3, d);
2109                         store_reg_to_var_flt(iptr->dst, d);
2110                         break;
2111
2112                 case ICMD_AALOAD:     /* ..., arrayref, index  ==> ..., value         */
2113
2114                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2115                         var_to_reg_int(s2, src, REG_ITMP2);
2116                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2117                         if (iptr->op1 == 0) {
2118                                 gen_nullptr_check(s1);
2119                                 gen_bound_check;
2120                         }
2121                         x86_64_mov_memindex_reg(cd, OFFSET(java_objectarray, data[0]), s1, s2, 3, d);
2122                         store_reg_to_var_int(iptr->dst, d);
2123                         break;
2124
2125
2126                 case ICMD_BASTORE:    /* ..., arrayref, index, value  ==> ...         */
2127
2128                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2129                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2130                         if (iptr->op1 == 0) {
2131                                 gen_nullptr_check(s1);
2132                                 gen_bound_check;
2133                         }
2134                         var_to_reg_int(s3, src, REG_ITMP3);
2135                         x86_64_movb_reg_memindex(cd, s3, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2136                         break;
2137
2138                 case ICMD_CASTORE:    /* ..., arrayref, index, value  ==> ...         */
2139
2140                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2141                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2142                         if (iptr->op1 == 0) {
2143                                 gen_nullptr_check(s1);
2144                                 gen_bound_check;
2145                         }
2146                         var_to_reg_int(s3, src, REG_ITMP3);
2147                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_chararray, data[0]), s1, s2, 1);
2148                         break;
2149
2150                 case ICMD_SASTORE:    /* ..., arrayref, index, value  ==> ...         */
2151
2152                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2153                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2154                         if (iptr->op1 == 0) {
2155                                 gen_nullptr_check(s1);
2156                                 gen_bound_check;
2157                         }
2158                         var_to_reg_int(s3, src, REG_ITMP3);
2159                         x86_64_movw_reg_memindex(cd, s3, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2160                         break;
2161
2162                 case ICMD_IASTORE:    /* ..., arrayref, index, value  ==> ...         */
2163
2164                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2165                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2166                         if (iptr->op1 == 0) {
2167                                 gen_nullptr_check(s1);
2168                                 gen_bound_check;
2169                         }
2170                         var_to_reg_int(s3, src, REG_ITMP3);
2171                         x86_64_movl_reg_memindex(cd, s3, OFFSET(java_intarray, data[0]), s1, s2, 2);
2172                         break;
2173
2174                 case ICMD_LASTORE:    /* ..., arrayref, index, value  ==> ...         */
2175
2176                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2177                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2178                         if (iptr->op1 == 0) {
2179                                 gen_nullptr_check(s1);
2180                                 gen_bound_check;
2181                         }
2182                         var_to_reg_int(s3, src, REG_ITMP3);
2183                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_longarray, data[0]), s1, s2, 3);
2184                         break;
2185
2186                 case ICMD_FASTORE:    /* ..., arrayref, index, value  ==> ...         */
2187
2188                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2189                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2190                         if (iptr->op1 == 0) {
2191                                 gen_nullptr_check(s1);
2192                                 gen_bound_check;
2193                         }
2194                         var_to_reg_flt(s3, src, REG_FTMP3);
2195                         x86_64_movss_reg_memindex(cd, s3, OFFSET(java_floatarray, data[0]), s1, s2, 2);
2196                         break;
2197
2198                 case ICMD_DASTORE:    /* ..., arrayref, index, value  ==> ...         */
2199
2200                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2201                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2202                         if (iptr->op1 == 0) {
2203                                 gen_nullptr_check(s1);
2204                                 gen_bound_check;
2205                         }
2206                         var_to_reg_flt(s3, src, REG_FTMP3);
2207                         x86_64_movsd_reg_memindex(cd, s3, OFFSET(java_doublearray, data[0]), s1, s2, 3);
2208                         break;
2209
2210                 case ICMD_AASTORE:    /* ..., arrayref, index, value  ==> ...         */
2211
2212                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2213                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2214                         if (iptr->op1 == 0) {
2215                                 gen_nullptr_check(s1);
2216                                 gen_bound_check;
2217                         }
2218                         var_to_reg_int(s3, src, REG_ITMP3);
2219
2220                         M_MOV(s1, rd->argintregs[0]);
2221                         M_MOV(s3, rd->argintregs[1]);
2222                         M_MOV_IMM(BUILTIN_canstore, REG_ITMP1);
2223                         M_CALL(REG_ITMP1);
2224                         M_TEST(REG_RESULT);
2225                         M_BEQ(0);
2226                         codegen_add_arraystoreexception_ref(cd, cd->mcodeptr);
2227
2228                         var_to_reg_int(s1, src->prev->prev, REG_ITMP1);
2229                         var_to_reg_int(s2, src->prev, REG_ITMP2);
2230                         var_to_reg_int(s3, src, REG_ITMP3);
2231                         x86_64_mov_reg_memindex(cd, s3, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2232                         break;
2233
2234
2235                 case ICMD_BASTORECONST: /* ..., arrayref, index  ==> ...              */
2236
2237                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2238                         var_to_reg_int(s2, src, REG_ITMP2);
2239                         if (iptr->op1 == 0) {
2240                                 gen_nullptr_check(s1);
2241                                 gen_bound_check;
2242                         }
2243                         x86_64_movb_imm_memindex(cd, iptr->val.i, OFFSET(java_bytearray, data[0]), s1, s2, 0);
2244                         break;
2245
2246                 case ICMD_CASTORECONST:   /* ..., arrayref, index  ==> ...            */
2247
2248                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2249                         var_to_reg_int(s2, src, REG_ITMP2);
2250                         if (iptr->op1 == 0) {
2251                                 gen_nullptr_check(s1);
2252                                 gen_bound_check;
2253                         }
2254                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_chararray, data[0]), s1, s2, 1);
2255                         break;
2256
2257                 case ICMD_SASTORECONST:   /* ..., arrayref, index  ==> ...            */
2258
2259                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2260                         var_to_reg_int(s2, src, REG_ITMP2);
2261                         if (iptr->op1 == 0) {
2262                                 gen_nullptr_check(s1);
2263                                 gen_bound_check;
2264                         }
2265                         x86_64_movw_imm_memindex(cd, iptr->val.i, OFFSET(java_shortarray, data[0]), s1, s2, 1);
2266                         break;
2267
2268                 case ICMD_IASTORECONST: /* ..., arrayref, index  ==> ...              */
2269
2270                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2271                         var_to_reg_int(s2, src, REG_ITMP2);
2272                         if (iptr->op1 == 0) {
2273                                 gen_nullptr_check(s1);
2274                                 gen_bound_check;
2275                         }
2276                         x86_64_movl_imm_memindex(cd, iptr->val.i, OFFSET(java_intarray, data[0]), s1, s2, 2);
2277                         break;
2278
2279                 case ICMD_LASTORECONST: /* ..., arrayref, index  ==> ...              */
2280
2281                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2282                         var_to_reg_int(s2, src, REG_ITMP2);
2283                         if (iptr->op1 == 0) {
2284                                 gen_nullptr_check(s1);
2285                                 gen_bound_check;
2286                         }
2287
2288                         if (IS_IMM32(iptr->val.l)) {
2289                                 x86_64_mov_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2290                         } else {
2291                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l & 0x00000000ffffffff), OFFSET(java_longarray, data[0]), s1, s2, 3);
2292                                 x86_64_movl_imm_memindex(cd, (u4) (iptr->val.l >> 32), OFFSET(java_longarray, data[0]) + 4, s1, s2, 3);
2293                         }
2294                         break;
2295
2296                 case ICMD_AASTORECONST: /* ..., arrayref, index  ==> ...              */
2297
2298                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2299                         var_to_reg_int(s2, src, REG_ITMP2);
2300                         if (iptr->op1 == 0) {
2301                                 gen_nullptr_check(s1);
2302                                 gen_bound_check;
2303                         }
2304                         x86_64_mov_imm_memindex(cd, 0, OFFSET(java_objectarray, data[0]), s1, s2, 3);
2305                         break;
2306
2307
2308                 case ICMD_GETSTATIC:  /* ...  ==> ..., value                          */
2309                                       /* op1 = type, val.a = field address            */
2310
2311                         if (iptr->val.a == NULL) {
2312                                 disp = dseg_addaddress(cd, NULL);
2313
2314 /*                              PROFILE_CYCLE_STOP; */
2315
2316                                 codegen_addpatchref(cd, cd->mcodeptr,
2317                                                                         PATCHER_get_putstatic,
2318                                                                         (unresolved_field *) iptr->target, disp);
2319
2320                                 if (opt_showdisassemble) {
2321                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2322                                 }
2323
2324 /*                              PROFILE_CYCLE_START; */
2325
2326                         } else {
2327                                 fieldinfo *fi = iptr->val.a;
2328
2329                                 disp = dseg_addaddress(cd, &(fi->value));
2330
2331                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2332                                         PROFILE_CYCLE_STOP;
2333
2334                                         codegen_addpatchref(cd, cd->mcodeptr,
2335                                                                                 PATCHER_clinit, fi->class, 0);
2336
2337                                         if (opt_showdisassemble) {
2338                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2339                                         }
2340
2341                                         PROFILE_CYCLE_START;
2342                                 }
2343                         }
2344
2345                         /* This approach is much faster than moving the field
2346                            address inline into a register. */
2347
2348                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2349                                                                         (ptrint) cd->mcodebase) + disp);
2350
2351                         switch (iptr->op1) {
2352                         case TYPE_INT:
2353                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2354                                 M_ILD(d, REG_ITMP2, 0);
2355                                 store_reg_to_var_int(iptr->dst, d);
2356                                 break;
2357                         case TYPE_LNG:
2358                         case TYPE_ADR:
2359                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2360                                 M_LLD(d, REG_ITMP2, 0);
2361                                 store_reg_to_var_int(iptr->dst, d);
2362                                 break;
2363                         case TYPE_FLT:
2364                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2365                                 x86_64_movss_membase_reg(cd, REG_ITMP2, 0, d);
2366                                 store_reg_to_var_flt(iptr->dst, d);
2367                                 break;
2368                         case TYPE_DBL:
2369                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2370                                 x86_64_movsd_membase_reg(cd, REG_ITMP2, 0, d);
2371                                 store_reg_to_var_flt(iptr->dst, d);
2372                                 break;
2373                         }
2374                         break;
2375
2376                 case ICMD_PUTSTATIC:  /* ..., value  ==> ...                          */
2377                                       /* op1 = type, val.a = field address            */
2378
2379                         if (iptr->val.a == NULL) {
2380                                 disp = dseg_addaddress(cd, NULL);
2381
2382 /*                              PROFILE_CYCLE_STOP; */
2383
2384                                 codegen_addpatchref(cd, cd->mcodeptr,
2385                                                                         PATCHER_get_putstatic,
2386                                                                         (unresolved_field *) iptr->target, disp);
2387
2388                                 if (opt_showdisassemble) {
2389                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2390                                 }
2391
2392 /*                              PROFILE_CYCLE_START; */
2393
2394                         } else {
2395                                 fieldinfo *fi = iptr->val.a;
2396
2397                                 disp = dseg_addaddress(cd, &(fi->value));
2398
2399                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2400                                         PROFILE_CYCLE_STOP;
2401
2402                                         codegen_addpatchref(cd, cd->mcodeptr,
2403                                                                                 PATCHER_clinit, fi->class, 0);
2404
2405                                         if (opt_showdisassemble) {
2406                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2407                                         }
2408
2409                                         PROFILE_CYCLE_START;
2410                                 }
2411                         }
2412
2413                         /* This approach is much faster than moving the field
2414                            address inline into a register. */
2415
2416                         M_ALD(REG_ITMP2, RIP, -(((ptrint) cd->mcodeptr + 7) -
2417                                                                         (ptrint) cd->mcodebase) + disp);
2418
2419                         switch (iptr->op1) {
2420                         case TYPE_INT:
2421                                 var_to_reg_int(s2, src, REG_ITMP1);
2422                                 M_IST(s2, REG_ITMP2, 0);
2423                                 break;
2424                         case TYPE_LNG:
2425                         case TYPE_ADR:
2426                                 var_to_reg_int(s2, src, REG_ITMP1);
2427                                 M_LST(s2, REG_ITMP2, 0);
2428                                 break;
2429                         case TYPE_FLT:
2430                                 var_to_reg_flt(s2, src, REG_FTMP1);
2431                                 x86_64_movss_reg_membase(cd, s2, REG_ITMP2, 0);
2432                                 break;
2433                         case TYPE_DBL:
2434                                 var_to_reg_flt(s2, src, REG_FTMP1);
2435                                 x86_64_movsd_reg_membase(cd, s2, REG_ITMP2, 0);
2436                                 break;
2437                         }
2438                         break;
2439
2440                 case ICMD_PUTSTATICCONST: /* ...  ==> ...                             */
2441                                           /* val = value (in current instruction)     */
2442                                           /* op1 = type, val.a = field address (in    */
2443                                           /* following NOP)                           */
2444
2445                         if (iptr[1].val.a == NULL) {
2446                                 disp = dseg_addaddress(cd, NULL);
2447
2448 /*                              PROFILE_CYCLE_STOP; */
2449
2450                                 codegen_addpatchref(cd, cd->mcodeptr,
2451                                                                         PATCHER_get_putstatic,
2452                                                                         (unresolved_field *) iptr[1].target, disp);
2453
2454                                 if (opt_showdisassemble) {
2455                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2456                                 }
2457
2458 /*                              PROFILE_CYCLE_START; */
2459
2460                         } else {
2461                                 fieldinfo *fi = iptr[1].val.a;
2462
2463                                 disp = dseg_addaddress(cd, &(fi->value));
2464
2465                                 if (!CLASS_IS_OR_ALMOST_INITIALIZED(fi->class)) {
2466                                         PROFILE_CYCLE_STOP;
2467
2468                                         codegen_addpatchref(cd, cd->mcodeptr,
2469                                                                                 PATCHER_clinit, fi->class, 0);
2470
2471                                         if (opt_showdisassemble) {
2472                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2473                                         }
2474
2475                                         PROFILE_CYCLE_START;
2476                                 }
2477                         }
2478
2479                         /* This approach is much faster than moving the field
2480                            address inline into a register. */
2481
2482                         M_ALD(REG_ITMP1, RIP, -(((ptrint) cd->mcodeptr + 7) -
2483                                                                         (ptrint) cd->mcodebase) + disp);
2484
2485                         switch (iptr->op1) {
2486                         case TYPE_INT:
2487                         case TYPE_FLT:
2488                                 M_IST_IMM(iptr->val.i, REG_ITMP1, 0);
2489                                 break;
2490                         case TYPE_LNG:
2491                         case TYPE_ADR:
2492                         case TYPE_DBL:
2493                                 if (IS_IMM32(iptr->val.l)) {
2494                                         M_LST_IMM32(iptr->val.l, REG_ITMP1, 0);
2495                                 } else {
2496                                         M_IST_IMM(iptr->val.l, REG_ITMP1, 0);
2497                                         M_IST_IMM(iptr->val.l >> 32, REG_ITMP1, 4);
2498                                 }
2499                                 break;
2500                         }
2501                         break;
2502
2503                 case ICMD_GETFIELD:   /* ...  ==> ..., value                          */
2504                                       /* op1 = type, val.i = field offset             */
2505
2506                         var_to_reg_int(s1, src, REG_ITMP1);
2507                         gen_nullptr_check(s1);
2508
2509                         if (iptr->val.a == NULL) {
2510 /*                              PROFILE_CYCLE_STOP; */
2511
2512                                 codegen_addpatchref(cd, cd->mcodeptr,
2513                                                                         PATCHER_get_putfield,
2514                                                                         (unresolved_field *) iptr->target, 0);
2515
2516                                 if (opt_showdisassemble) {
2517                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2518                                 }
2519
2520 /*                              PROFILE_CYCLE_START; */
2521
2522                                 disp = 0;
2523
2524                         } else {
2525                                 disp = ((fieldinfo *) (iptr->val.a))->offset;
2526                         }
2527
2528                         switch (iptr->op1) {
2529                         case TYPE_INT:
2530                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2531                                 if (iptr->val.a == NULL)
2532                                         M_ILD32(d, s1, disp);
2533                                 else
2534                                         M_ILD(d, s1, disp);
2535                                 store_reg_to_var_int(iptr->dst, d);
2536                                 break;
2537                         case TYPE_LNG:
2538                         case TYPE_ADR:
2539                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
2540                                 if (iptr->val.a == NULL)
2541                                         M_LLD32(d, s1, disp);
2542                                 else
2543                                         M_LLD(d, s1, disp);
2544                                 store_reg_to_var_int(iptr->dst, d);
2545                                 break;
2546                         case TYPE_FLT:
2547                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2548                                 x86_64_movss_membase32_reg(cd, s1, disp, d);
2549                                 store_reg_to_var_flt(iptr->dst, d);
2550                                 break;
2551                         case TYPE_DBL:
2552                                 d = reg_of_var(rd, iptr->dst, REG_FTMP1);
2553                                 x86_64_movsd_membase32_reg(cd, s1, disp, d);
2554                                 store_reg_to_var_flt(iptr->dst, d);
2555                                 break;
2556                         }
2557                         break;
2558
2559                 case ICMD_PUTFIELD:   /* ..., objectref, value  ==> ...               */
2560                                       /* op1 = type, val.i = field offset             */
2561
2562                         var_to_reg_int(s1, src->prev, REG_ITMP1);
2563                         gen_nullptr_check(s1);
2564
2565                         if (IS_INT_LNG_TYPE(iptr->op1)) {
2566                                 var_to_reg_int(s2, src, REG_ITMP2);
2567                         } else {
2568                                 var_to_reg_flt(s2, src, REG_FTMP2);
2569                         }
2570
2571                         if (iptr->val.a == NULL) {
2572 /*                              PROFILE_CYCLE_STOP; */
2573
2574                                 codegen_addpatchref(cd, cd->mcodeptr,
2575                                                                         PATCHER_get_putfield,
2576                                                                         (unresolved_field *) iptr->target, 0);
2577
2578                                 if (opt_showdisassemble) {
2579                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2580                                 }
2581
2582 /*                              PROFILE_CYCLE_START; */
2583
2584                                 disp = 0;
2585
2586                         } else {
2587                                 disp = ((fieldinfo *) (iptr->val.a))->offset;
2588                         }
2589
2590                         switch (iptr->op1) {
2591                         case TYPE_INT:
2592                                 if (iptr->val.a == NULL)
2593                                         M_IST32(s2, s1, disp);
2594                                 else
2595                                         M_IST(s2, s1, disp);
2596                                 break;
2597                         case TYPE_LNG:
2598                         case TYPE_ADR:
2599                                 if (iptr->val.a == NULL)
2600                                         M_LST32(s2, s1, disp);
2601                                 else
2602                                         M_LST(s2, s1, disp);
2603                                 break;
2604                         case TYPE_FLT:
2605                                 x86_64_movss_reg_membase32(cd, s2, s1, disp);
2606                                 break;
2607                         case TYPE_DBL:
2608                                 x86_64_movsd_reg_membase32(cd, s2, s1, disp);
2609                                 break;
2610                         }
2611                         break;
2612
2613                 case ICMD_PUTFIELDCONST:  /* ..., objectref, value  ==> ...           */
2614                                           /* val = value (in current instruction)     */
2615                                           /* op1 = type, val.a = field address (in    */
2616                                           /* following NOP)                           */
2617
2618                         var_to_reg_int(s1, src, REG_ITMP1);
2619                         gen_nullptr_check(s1);
2620
2621                         if (iptr[1].val.a == NULL) {
2622 /*                              PROFILE_CYCLE_STOP; */
2623
2624                                 codegen_addpatchref(cd, cd->mcodeptr,
2625                                                                         PATCHER_putfieldconst,
2626                                                                         (unresolved_field *) iptr[1].target, 0);
2627
2628                                 if (opt_showdisassemble) {
2629                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2630                                 }
2631
2632 /*                              PROFILE_CYCLE_START; */
2633
2634                                 disp = 0;
2635
2636                         } else {
2637                                 disp = ((fieldinfo *) (iptr[1].val.a))->offset;
2638                         }
2639
2640                         switch (iptr->op1) {
2641                         case TYPE_INT:
2642                         case TYPE_FLT:
2643                                 if (iptr[1].val.a == NULL)
2644                                         M_IST32_IMM(iptr->val.i, s1, disp);
2645                                 else
2646                                         M_IST_IMM(iptr->val.i, s1, disp);
2647                                 break;
2648                         case TYPE_LNG:
2649                         case TYPE_ADR:
2650                         case TYPE_DBL:
2651                                 /* We can only optimize the move, if the class is
2652                                    resolved. Otherwise we don't know what to patch. */
2653                                 if (iptr[1].val.a == NULL) {
2654                                         M_IST32_IMM(iptr->val.l, s1, disp);
2655                                         M_IST32_IMM(iptr->val.l >> 32, s1, disp + 4);
2656                                 } else {
2657                                         if (IS_IMM32(iptr->val.l)) {
2658                                                 M_LST_IMM32(iptr->val.l, s1, disp);
2659                                         } else {
2660                                                 M_IST_IMM(iptr->val.l, s1, disp);
2661                                                 M_IST_IMM(iptr->val.l >> 32, s1, disp + 4);
2662                                         }
2663                                 }
2664                                 break;
2665                         }
2666                         break;
2667
2668
2669                 /* branch operations **************************************************/
2670
2671                 case ICMD_ATHROW:       /* ..., objectref ==> ... (, objectref)       */
2672
2673                         var_to_reg_int(s1, src, REG_ITMP1);
2674                         M_INTMOVE(s1, REG_ITMP1_XPTR);
2675
2676                         PROFILE_CYCLE_STOP;
2677
2678 #ifdef ENABLE_VERIFIER
2679                         if (iptr->val.a) {
2680                                 codegen_addpatchref(cd, cd->mcodeptr,
2681                                                                         PATCHER_athrow_areturn,
2682                                                                         (unresolved_class *) iptr->val.a, 0);
2683
2684                                 if (opt_showdisassemble) {
2685                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2686                                 }
2687                         }
2688 #endif /* ENABLE_VERIFIER */
2689
2690                         M_CALL_IMM(0);                            /* passing exception pc */
2691                         M_POP(REG_ITMP2_XPC);
2692
2693                         M_MOV_IMM(asm_handle_exception, REG_ITMP3);
2694                         M_JMP(REG_ITMP3);
2695                         break;
2696
2697                 case ICMD_GOTO:         /* ... ==> ...                                */
2698                                         /* op1 = target JavaVM pc                     */
2699
2700                         M_JMP_IMM(0);
2701                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2702                         break;
2703
2704                 case ICMD_JSR:          /* ... ==> ...                                */
2705                                         /* op1 = target JavaVM pc                     */
2706
2707                         M_CALL_IMM(0);
2708                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2709                         break;
2710
2711                 case ICMD_RET:          /* ... ==> ...                                */
2712                                         /* op1 = local variable                       */
2713
2714                         var = &(rd->locals[iptr->op1][TYPE_ADR]);
2715                         var_to_reg_int(s1, var, REG_ITMP1);
2716                         M_JMP(s1);
2717                         break;
2718
2719                 case ICMD_IFNULL:       /* ..., value ==> ...                         */
2720                                         /* op1 = target JavaVM pc                     */
2721
2722                         if (src->flags & INMEMORY)
2723                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2724                         else
2725                                 M_TEST(src->regoff);
2726                         M_BEQ(0);
2727                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2728                         break;
2729
2730                 case ICMD_IFNONNULL:    /* ..., value ==> ...                         */
2731                                         /* op1 = target JavaVM pc                     */
2732
2733                         if (src->flags & INMEMORY)
2734                                 M_CMP_IMM_MEMBASE(0, REG_SP, src->regoff * 8);
2735                         else
2736                                 M_TEST(src->regoff);
2737                         M_BNE(0);
2738                         codegen_addreference(cd, (basicblock *) iptr->target, cd->mcodeptr);
2739                         break;
2740
2741                 case ICMD_IFEQ:         /* ..., value ==> ...                         */
2742                                         /* op1 = target JavaVM pc, val.i = constant   */
2743
2744                         x86_64_emit_ifcc(cd, X86_64_CC_E, src, iptr);
2745                         break;
2746
2747                 case ICMD_IFLT:         /* ..., value ==> ...                         */
2748                                         /* op1 = target JavaVM pc, val.i = constant   */
2749
2750                         x86_64_emit_ifcc(cd, X86_64_CC_L, src, iptr);
2751                         break;
2752
2753                 case ICMD_IFLE:         /* ..., value ==> ...                         */
2754                                         /* op1 = target JavaVM pc, val.i = constant   */
2755
2756                         x86_64_emit_ifcc(cd, X86_64_CC_LE, src, iptr);
2757                         break;
2758
2759                 case ICMD_IFNE:         /* ..., value ==> ...                         */
2760                                         /* op1 = target JavaVM pc, val.i = constant   */
2761
2762                         x86_64_emit_ifcc(cd, X86_64_CC_NE, src, iptr);
2763                         break;
2764
2765                 case ICMD_IFGT:         /* ..., value ==> ...                         */
2766                                         /* op1 = target JavaVM pc, val.i = constant   */
2767
2768                         x86_64_emit_ifcc(cd, X86_64_CC_G, src, iptr);
2769                         break;
2770
2771                 case ICMD_IFGE:         /* ..., value ==> ...                         */
2772                                         /* op1 = target JavaVM pc, val.i = constant   */
2773
2774                         x86_64_emit_ifcc(cd, X86_64_CC_GE, src, iptr);
2775                         break;
2776
2777                 case ICMD_IF_LEQ:       /* ..., value ==> ...                         */
2778                                         /* op1 = target JavaVM pc, val.l = constant   */
2779
2780                         x86_64_emit_if_lcc(cd, X86_64_CC_E, src, iptr);
2781                         break;
2782
2783                 case ICMD_IF_LLT:       /* ..., value ==> ...                         */
2784                                         /* op1 = target JavaVM pc, val.l = constant   */
2785
2786                         x86_64_emit_if_lcc(cd, X86_64_CC_L, src, iptr);
2787                         break;
2788
2789                 case ICMD_IF_LLE:       /* ..., value ==> ...                         */
2790                                         /* op1 = target JavaVM pc, val.l = constant   */
2791
2792                         x86_64_emit_if_lcc(cd, X86_64_CC_LE, src, iptr);
2793                         break;
2794
2795                 case ICMD_IF_LNE:       /* ..., value ==> ...                         */
2796                                         /* op1 = target JavaVM pc, val.l = constant   */
2797
2798                         x86_64_emit_if_lcc(cd, X86_64_CC_NE, src, iptr);
2799                         break;
2800
2801                 case ICMD_IF_LGT:       /* ..., value ==> ...                         */
2802                                         /* op1 = target JavaVM pc, val.l = constant   */
2803
2804                         x86_64_emit_if_lcc(cd, X86_64_CC_G, src, iptr);
2805                         break;
2806
2807                 case ICMD_IF_LGE:       /* ..., value ==> ...                         */
2808                                         /* op1 = target JavaVM pc, val.l = constant   */
2809
2810                         x86_64_emit_if_lcc(cd, X86_64_CC_GE, src, iptr);
2811                         break;
2812
2813                 case ICMD_IF_ICMPEQ:    /* ..., value, value ==> ...                  */
2814                                         /* op1 = target JavaVM pc                     */
2815
2816                         x86_64_emit_if_icmpcc(cd, X86_64_CC_E, src, iptr);
2817                         break;
2818
2819                 case ICMD_IF_LCMPEQ:    /* ..., value, value ==> ...                  */
2820                 case ICMD_IF_ACMPEQ:    /* op1 = target JavaVM pc                     */
2821
2822                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_E, src, iptr);
2823                         break;
2824
2825                 case ICMD_IF_ICMPNE:    /* ..., value, value ==> ...                  */
2826                                         /* op1 = target JavaVM pc                     */
2827
2828                         x86_64_emit_if_icmpcc(cd, X86_64_CC_NE, src, iptr);
2829                         break;
2830
2831                 case ICMD_IF_LCMPNE:    /* ..., value, value ==> ...                  */
2832                 case ICMD_IF_ACMPNE:    /* op1 = target JavaVM pc                     */
2833
2834                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_NE, src, iptr);
2835                         break;
2836
2837                 case ICMD_IF_ICMPLT:    /* ..., value, value ==> ...                  */
2838                                         /* op1 = target JavaVM pc                     */
2839
2840                         x86_64_emit_if_icmpcc(cd, X86_64_CC_L, src, iptr);
2841                         break;
2842
2843                 case ICMD_IF_LCMPLT:    /* ..., value, value ==> ...                  */
2844                                     /* op1 = target JavaVM pc                     */
2845
2846                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_L, src, iptr);
2847                         break;
2848
2849                 case ICMD_IF_ICMPGT:    /* ..., value, value ==> ...                  */
2850                                         /* op1 = target JavaVM pc                     */
2851
2852                         x86_64_emit_if_icmpcc(cd, X86_64_CC_G, src, iptr);
2853                         break;
2854
2855                 case ICMD_IF_LCMPGT:    /* ..., value, value ==> ...                  */
2856                                 /* op1 = target JavaVM pc                     */
2857
2858                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_G, src, iptr);
2859                         break;
2860
2861                 case ICMD_IF_ICMPLE:    /* ..., value, value ==> ...                  */
2862                                         /* op1 = target JavaVM pc                     */
2863
2864                         x86_64_emit_if_icmpcc(cd, X86_64_CC_LE, src, iptr);
2865                         break;
2866
2867                 case ICMD_IF_LCMPLE:    /* ..., value, value ==> ...                  */
2868                                         /* op1 = target JavaVM pc                     */
2869
2870                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_LE, src, iptr);
2871                         break;
2872
2873                 case ICMD_IF_ICMPGE:    /* ..., value, value ==> ...                  */
2874                                         /* op1 = target JavaVM pc                     */
2875
2876                         x86_64_emit_if_icmpcc(cd, X86_64_CC_GE, src, iptr);
2877                         break;
2878
2879                 case ICMD_IF_LCMPGE:    /* ..., value, value ==> ...                  */
2880                                     /* op1 = target JavaVM pc                     */
2881
2882                         x86_64_emit_if_lcmpcc(cd, X86_64_CC_GE, src, iptr);
2883                         break;
2884
2885                 /* (value xx 0) ? IFxx_ICONST : ELSE_ICONST                           */
2886
2887                 case ICMD_ELSE_ICONST:  /* handled by IFxx_ICONST                     */
2888                         break;
2889
2890                 case ICMD_IFEQ_ICONST:  /* ..., value ==> ..., constant               */
2891                 case ICMD_IFNE_ICONST:  /* val.i = constant                           */
2892                 case ICMD_IFLT_ICONST:
2893                 case ICMD_IFGE_ICONST:
2894                 case ICMD_IFGT_ICONST:
2895                 case ICMD_IFLE_ICONST:
2896
2897                         var_to_reg_int(s1, src, REG_ITMP1);
2898                         d = reg_of_var(rd, iptr->dst, REG_ITMP3);
2899                         if (iptr[1].opc == ICMD_ELSE_ICONST) {
2900                                 if (s1 == d) {
2901                                         M_INTMOVE(s1, REG_ITMP1);
2902                                         s1 = REG_ITMP1;
2903                                 }
2904                                 if (iptr[1].val.i == 0)
2905                                         M_CLR(d);
2906                                 else
2907                                         M_IMOV_IMM(iptr[1].val.i, d);
2908                         }
2909                         if (iptr->val.i == 0)
2910                                 M_CLR(REG_ITMP2);
2911                         else
2912                                 M_IMOV_IMM(iptr->val.i, REG_ITMP2);
2913                         M_ITEST(s1);
2914
2915                         switch (iptr->opc) {
2916                         case ICMD_IFEQ_ICONST:
2917                                 M_CMOVEQ(REG_ITMP2, d);
2918                                 break;
2919                         case ICMD_IFNE_ICONST:
2920                                 M_CMOVNE(REG_ITMP2, d);
2921                                 break;
2922                         case ICMD_IFLT_ICONST:
2923                                 M_CMOVLT(REG_ITMP2, d);
2924                                 break;
2925                         case ICMD_IFGE_ICONST:
2926                                 M_CMOVGE(REG_ITMP2, d);
2927                                 break;
2928                         case ICMD_IFGT_ICONST:
2929                                 M_CMOVGT(REG_ITMP2, d);
2930                                 break;
2931                         case ICMD_IFLE_ICONST:
2932                                 M_CMOVLE(REG_ITMP2, d);
2933                                 break;
2934                         }
2935
2936                         store_reg_to_var_int(iptr->dst, d);
2937                         break;
2938
2939
2940                 case ICMD_IRETURN:      /* ..., retvalue ==> ...                      */
2941                 case ICMD_LRETURN:
2942
2943                         var_to_reg_int(s1, src, REG_RESULT);
2944                         M_INTMOVE(s1, REG_RESULT);
2945                         goto nowperformreturn;
2946
2947                 case ICMD_ARETURN:      /* ..., retvalue ==> ...                      */
2948
2949                         var_to_reg_int(s1, src, REG_RESULT);
2950                         M_INTMOVE(s1, REG_RESULT);
2951
2952 #ifdef ENABLE_VERIFIER
2953                         if (iptr->val.a) {
2954                                 PROFILE_CYCLE_STOP;
2955
2956                                 codegen_addpatchref(cd, cd->mcodeptr,
2957                                                                         PATCHER_athrow_areturn,
2958                                                                         (unresolved_class *) iptr->val.a, 0);
2959
2960                                 if (opt_showdisassemble) {
2961                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
2962                                 }
2963
2964                                 PROFILE_CYCLE_START;
2965                         }
2966 #endif /* ENABLE_VERIFIER */
2967                         goto nowperformreturn;
2968
2969                 case ICMD_FRETURN:      /* ..., retvalue ==> ...                      */
2970                 case ICMD_DRETURN:
2971
2972                         var_to_reg_flt(s1, src, REG_FRESULT);
2973                         M_FLTMOVE(s1, REG_FRESULT);
2974                         goto nowperformreturn;
2975
2976                 case ICMD_RETURN:      /* ...  ==> ...                                */
2977
2978 nowperformreturn:
2979                         {
2980                         s4 i, p;
2981
2982                         p = parentargs_base;
2983
2984 #if !defined(NDEBUG)
2985                         /* generate call trace */
2986
2987                         if (opt_verbosecall) {
2988                                 x86_64_alu_imm_reg(cd, X86_64_SUB, 2 * 8, REG_SP);
2989
2990                                 x86_64_mov_reg_membase(cd, REG_RESULT, REG_SP, 0 * 8);
2991                                 x86_64_movq_reg_membase(cd, REG_FRESULT, REG_SP, 1 * 8);
2992
2993                                 x86_64_mov_imm_reg(cd, (u8) m, rd->argintregs[0]);
2994                                 x86_64_mov_reg_reg(cd, REG_RESULT, rd->argintregs[1]);
2995                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
2996                                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
2997
2998                                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
2999                                 M_CALL(REG_ITMP1);
3000
3001                                 x86_64_mov_membase_reg(cd, REG_SP, 0 * 8, REG_RESULT);
3002                                 x86_64_movq_membase_reg(cd, REG_SP, 1 * 8, REG_FRESULT);
3003
3004                                 x86_64_alu_imm_reg(cd, X86_64_ADD, 2 * 8, REG_SP);
3005                         }
3006 #endif /* !defined(NDEBUG) */
3007
3008 #if defined(USE_THREADS)
3009                         if (checksync && (m->flags & ACC_SYNCHRONIZED)) {
3010                                 M_ALD(rd->argintregs[0], REG_SP, rd->memuse * 8);
3011
3012                                 /* we need to save the proper return value */
3013                                 switch (iptr->opc) {
3014                                 case ICMD_IRETURN:
3015                                 case ICMD_ARETURN:
3016                                 case ICMD_LRETURN:
3017                                         M_LST(REG_RESULT, REG_SP, rd->memuse * 8);
3018                                         break;
3019                                 case ICMD_FRETURN:
3020                                 case ICMD_DRETURN:
3021                                         M_DST(REG_FRESULT, REG_SP, rd->memuse * 8);
3022                                         break;
3023                                 }
3024
3025                                 M_MOV_IMM(builtin_monitorexit, REG_ITMP1);
3026                                 M_CALL(REG_ITMP1);
3027
3028                                 /* and now restore the proper return value */
3029                                 switch (iptr->opc) {
3030                                 case ICMD_IRETURN:
3031                                 case ICMD_ARETURN:
3032                                 case ICMD_LRETURN:
3033                                         M_LLD(REG_RESULT, REG_SP, rd->memuse * 8);
3034                                         break;
3035                                 case ICMD_FRETURN:
3036                                 case ICMD_DRETURN:
3037                                         M_DLD(REG_FRESULT, REG_SP, rd->memuse * 8);
3038                                         break;
3039                                 }
3040                         }
3041 #endif
3042
3043                         /* restore saved registers */
3044
3045                         for (i = INT_SAV_CNT - 1; i >= rd->savintreguse; i--) {
3046                                 p--; M_LLD(rd->savintregs[i], REG_SP, p * 8);
3047                         }
3048                         for (i = FLT_SAV_CNT - 1; i >= rd->savfltreguse; i--) {
3049                                 p--; M_DLD(rd->savfltregs[i], REG_SP, p * 8);
3050                         }
3051
3052                         /* deallocate stack */
3053
3054                         if (parentargs_base)
3055                                 M_AADD_IMM(parentargs_base * 8, REG_SP);
3056
3057                         /* generate method profiling code */
3058
3059                         PROFILE_CYCLE_STOP;
3060
3061                         M_RET;
3062                         }
3063                         break;
3064
3065
3066                 case ICMD_TABLESWITCH:  /* ..., index ==> ...                         */
3067                         {
3068                                 s4 i, l, *s4ptr;
3069                                 void **tptr;
3070
3071                                 tptr = (void **) iptr->target;
3072
3073                                 s4ptr = iptr->val.a;
3074                                 l = s4ptr[1];                          /* low     */
3075                                 i = s4ptr[2];                          /* high    */
3076
3077                                 var_to_reg_int(s1, src, REG_ITMP1);
3078                                 M_INTMOVE(s1, REG_ITMP1);
3079                                 if (l != 0) {
3080                                         x86_64_alul_imm_reg(cd, X86_64_SUB, l, REG_ITMP1);
3081                                 }
3082                                 i = i - l + 1;
3083
3084                 /* range check */
3085                                 x86_64_alul_imm_reg(cd, X86_64_CMP, i - 1, REG_ITMP1);
3086                                 x86_64_jcc(cd, X86_64_CC_A, 0);
3087
3088                 /* codegen_addreference(cd, BlockPtrOfPC(s4ptr[0]), cd->mcodeptr); */
3089                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3090
3091                                 /* build jump table top down and use address of lowest entry */
3092
3093                 /* s4ptr += 3 + i; */
3094                                 tptr += i;
3095
3096                                 while (--i >= 0) {
3097                                         dseg_addtarget(cd, (basicblock *) tptr[0]);
3098                                         --tptr;
3099                                 }
3100
3101                                 /* length of dataseg after last dseg_addtarget is used by load */
3102
3103                                 x86_64_mov_imm_reg(cd, 0, REG_ITMP2);
3104                                 dseg_adddata(cd, cd->mcodeptr);
3105                                 x86_64_mov_memindex_reg(cd, -(cd->dseglen), REG_ITMP2, REG_ITMP1, 3, REG_ITMP1);
3106                                 x86_64_jmp_reg(cd, REG_ITMP1);
3107                         }
3108                         break;
3109
3110
3111                 case ICMD_LOOKUPSWITCH: /* ..., key ==> ...                           */
3112                         {
3113                                 s4 i, l, val, *s4ptr;
3114                                 void **tptr;
3115
3116                                 tptr = (void **) iptr->target;
3117
3118                                 s4ptr = iptr->val.a;
3119                                 l = s4ptr[0];                          /* default  */
3120                                 i = s4ptr[1];                          /* count    */
3121
3122                                 MCODECHECK(8 + ((7 + 6) * i) + 5);
3123                                 var_to_reg_int(s1, src, REG_ITMP1);    /* reg compare should always be faster */
3124                                 while (--i >= 0) {
3125                                         s4ptr += 2;
3126                                         ++tptr;
3127
3128                                         val = s4ptr[0];
3129                                         x86_64_alul_imm_reg(cd, X86_64_CMP, val, s1);
3130                                         x86_64_jcc(cd, X86_64_CC_E, 0);
3131                                         codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3132                                 }
3133
3134                                 x86_64_jmp_imm(cd, 0);
3135
3136                                 tptr = (void **) iptr->target;
3137                                 codegen_addreference(cd, (basicblock *) tptr[0], cd->mcodeptr);
3138                         }
3139                         break;
3140
3141
3142                 case ICMD_BUILTIN:      /* ..., [arg1, [arg2 ...]] ==> ...            */
3143                                         /* op1 = arg count val.a = builtintable entry */
3144
3145                         bte = iptr->val.a;
3146                         md = bte->md;
3147                         goto gen_method;
3148
3149                 case ICMD_INVOKESTATIC: /* ..., [arg1, [arg2 ...]] ==> ...            */
3150                                         /* op1 = arg count, val.a = method pointer    */
3151
3152                 case ICMD_INVOKESPECIAL:/* ..., objectref, [arg1, [arg2 ...]] ==> ... */
3153                 case ICMD_INVOKEVIRTUAL:/* op1 = arg count, val.a = method pointer    */
3154                 case ICMD_INVOKEINTERFACE:
3155
3156                         lm = iptr->val.a;
3157
3158                         if (lm == NULL) {
3159                                 unresolved_method *um = iptr->target;
3160                                 md = um->methodref->parseddesc.md;
3161                         } else {
3162                                 md = lm->parseddesc;
3163                         }
3164
3165 gen_method:
3166                         s3 = md->paramcount;
3167
3168                         MCODECHECK((20 * s3) + 128);
3169
3170                         /* copy arguments to registers or stack location */
3171
3172                         for (s3 = s3 - 1; s3 >= 0; s3--, src = src->prev) {
3173                                 if (src->varkind == ARGVAR)
3174                                         continue;
3175                                 if (IS_INT_LNG_TYPE(src->type)) {
3176                                         if (!md->params[s3].inmemory) {
3177                                                 s1 = rd->argintregs[md->params[s3].regoff];
3178                                                 var_to_reg_int(d, src, s1);
3179                                                 M_INTMOVE(d, s1);
3180                                         } else {
3181                                                 var_to_reg_int(d, src, REG_ITMP1);
3182                                                 M_LST(d, REG_SP, md->params[s3].regoff * 8);
3183                                         }
3184
3185                                 } else {
3186                                         if (!md->params[s3].inmemory) {
3187                                                 s1 = rd->argfltregs[md->params[s3].regoff];
3188                                                 var_to_reg_flt(d, src, s1);
3189                                                 M_FLTMOVE(d, s1);
3190                                         } else {
3191                                                 var_to_reg_flt(d, src, REG_FTMP1);
3192                                                 M_DST(d, REG_SP, md->params[s3].regoff * 8);
3193                                         }
3194                                 }
3195                         }
3196
3197                         /* generate method profiling code */
3198
3199                         PROFILE_CYCLE_STOP;
3200
3201                         switch (iptr->opc) {
3202                         case ICMD_BUILTIN:
3203                                 a = (ptrint) bte->fp;
3204                                 d = md->returntype.type;
3205
3206                                 M_MOV_IMM(a, REG_ITMP1);
3207                                 M_CALL(REG_ITMP1);
3208
3209                                 /* if op1 == true, we need to check for an exception */
3210
3211                                 if (iptr->op1 == true) {
3212                                         M_TEST(REG_RESULT);
3213                                         M_BEQ(0);
3214                                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3215                                 }
3216                                 break;
3217
3218                         case ICMD_INVOKESPECIAL:
3219                                 M_TEST(rd->argintregs[0]);
3220                                 M_BEQ(0);
3221                                 codegen_add_nullpointerexception_ref(cd, cd->mcodeptr);
3222
3223                                 /* first argument contains pointer */
3224 /*                              gen_nullptr_check(rd->argintregs[0]); */
3225
3226                                 /* access memory for hardware nullptr */
3227 /*                              x86_64_mov_membase_reg(cd, rd->argintregs[0], 0, REG_ITMP2); */
3228
3229                                 /* fall through */
3230
3231                         case ICMD_INVOKESTATIC:
3232                                 if (lm == NULL) {
3233                                         unresolved_method *um = iptr->target;
3234
3235                                         codegen_addpatchref(cd, cd->mcodeptr,
3236                                                                                 PATCHER_invokestatic_special, um, 0);
3237
3238                                         if (opt_showdisassemble) {
3239                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3240                                         }
3241
3242                                         a = 0;
3243                                         d = um->methodref->parseddesc.md->returntype.type;
3244
3245                                 } else {
3246                                         a = (ptrint) lm->stubroutine;
3247                                         d = lm->parseddesc->returntype.type;
3248                                 }
3249
3250                                 M_MOV_IMM(a, REG_ITMP2);
3251                                 M_CALL(REG_ITMP2);
3252                                 break;
3253
3254                         case ICMD_INVOKEVIRTUAL:
3255                                 gen_nullptr_check(rd->argintregs[0]);
3256
3257                                 if (lm == NULL) {
3258                                         unresolved_method *um = iptr->target;
3259
3260                                         codegen_addpatchref(cd, cd->mcodeptr,
3261                                                                                 PATCHER_invokevirtual, um, 0);
3262
3263                                         if (opt_showdisassemble) {
3264                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3265                                         }
3266
3267                                         s1 = 0;
3268                                         d = um->methodref->parseddesc.md->returntype.type;
3269
3270                                 } else {
3271                                         s1 = OFFSET(vftbl_t, table[0]) +
3272                                                 sizeof(methodptr) * lm->vftblindex;
3273                                         d = lm->parseddesc->returntype.type;
3274                                 }
3275
3276                                 x86_64_mov_membase_reg(cd, rd->argintregs[0],
3277                                                                            OFFSET(java_objectheader, vftbl),
3278                                                                            REG_ITMP2);
3279                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP1);
3280                                 M_CALL(REG_ITMP1);
3281                                 break;
3282
3283                         case ICMD_INVOKEINTERFACE:
3284                                 gen_nullptr_check(rd->argintregs[0]);
3285
3286                                 if (lm == NULL) {
3287                                         unresolved_method *um = iptr->target;
3288
3289                                         codegen_addpatchref(cd, cd->mcodeptr,
3290                                                                                 PATCHER_invokeinterface, um, 0);
3291
3292                                         if (opt_showdisassemble) {
3293                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3294                                         }
3295
3296                                         s1 = 0;
3297                                         s2 = 0;
3298                                         d = um->methodref->parseddesc.md->returntype.type;
3299
3300                                 } else {
3301                                         s1 = OFFSET(vftbl_t, interfacetable[0]) -
3302                                                 sizeof(methodptr) * lm->class->index;
3303
3304                                         s2 = sizeof(methodptr) * (lm - lm->class->methods);
3305
3306                                         d = lm->parseddesc->returntype.type;
3307                                 }
3308
3309                                 M_ALD(REG_ITMP2, rd->argintregs[0],
3310                                           OFFSET(java_objectheader, vftbl));
3311                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s1, REG_ITMP2);
3312                                 x86_64_mov_membase32_reg(cd, REG_ITMP2, s2, REG_ITMP1);
3313                                 M_CALL(REG_ITMP1);
3314                                 break;
3315                         }
3316
3317                         /* generate method profiling code */
3318
3319                         PROFILE_CYCLE_START;
3320
3321                         /* d contains return type */
3322
3323                         if (d != TYPE_VOID) {
3324                                 if (IS_INT_LNG_TYPE(iptr->dst->type)) {
3325                                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3326                                         M_INTMOVE(REG_RESULT, s1);
3327                                         store_reg_to_var_int(iptr->dst, s1);
3328                                 } else {
3329                                         s1 = reg_of_var(rd, iptr->dst, REG_FRESULT);
3330                                         M_FLTMOVE(REG_FRESULT, s1);
3331                                         store_reg_to_var_flt(iptr->dst, s1);
3332                                 }
3333                         }
3334                         break;
3335
3336
3337                 case ICMD_CHECKCAST:  /* ..., objectref ==> ..., objectref            */
3338
3339                                       /* op1:   0 == array, 1 == class                */
3340                                       /* val.a: (classinfo *) superclass              */
3341
3342                         /*  superclass is an interface:
3343                          *
3344                          *  OK if ((sub == NULL) ||
3345                          *         (sub->vftbl->interfacetablelength > super->index) &&
3346                          *         (sub->vftbl->interfacetable[-super->index] != NULL));
3347                          *
3348                          *  superclass is a class:
3349                          *
3350                          *  OK if ((sub == NULL) || (0
3351                          *         <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3352                          *         super->vftbl->diffval));
3353                          */
3354
3355                         if (iptr->op1 == 1) {
3356                                 /* object type cast-check */
3357
3358                                 classinfo *super;
3359                                 vftbl_t   *supervftbl;
3360                                 s4         superindex;
3361
3362                                 super = (classinfo *) iptr->val.a;
3363
3364                                 if (!super) {
3365                                         superindex = 0;
3366                                         supervftbl = NULL;
3367
3368                                 } else {
3369                                         superindex = super->index;
3370                                         supervftbl = super->vftbl;
3371                                 }
3372
3373 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3374                                 codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3375 #endif
3376                                 var_to_reg_int(s1, src, REG_ITMP1);
3377
3378                                 /* calculate interface checkcast code size */
3379
3380                                 s2 = 3; /* mov_membase_reg */
3381                                 CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3382
3383                                 s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub imm32 */ +
3384                                         3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3385                                         3 /* test */ + 6 /* jcc */;
3386
3387                                 if (!super)
3388                                         s2 += (opt_showdisassemble ? 5 : 0);
3389
3390                                 /* calculate class checkcast code size */
3391
3392                                 s3 = 3; /* mov_membase_reg */
3393                                 CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3394                                 s3 += 10 /* mov_imm_reg */ + 3 + 4 /* movl_membase32_reg */;
3395
3396 #if 0
3397                                 if (s1 != REG_ITMP1) {
3398                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3399                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, baseval));
3400                                         a += 3;    /* movl_membase_reg - only if REG_ITMP3 == R11 */
3401                                         CALCOFFSETBYTES(a, REG_ITMP3, OFFSET(vftbl_t, diffval));
3402                                         a += 3;    /* sub */
3403
3404                                 } else
3405 #endif
3406                                         {
3407                                                 s3 += 3 + 4 /* movl_membase32_reg */ + 3 /* sub */ +
3408                                                         10 /* mov_imm_reg */ + 3 /* movl_membase_reg */;
3409                                                 CALCOFFSETBYTES(s3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3410                                         }
3411
3412                                 s3 += 3 /* cmp */ + 6 /* jcc */;
3413
3414                                 if (!super)
3415                                         s3 += (opt_showdisassemble ? 5 : 0);
3416
3417                                 /* if class is not resolved, check which code to call */
3418
3419                                 if (!super) {
3420                                         M_TEST(s1);
3421                                         M_BEQ(6 + (opt_showdisassemble ? 5 : 0) + 7 + 6 + s2 + 5 + s3);
3422
3423                                         codegen_addpatchref(cd, cd->mcodeptr,
3424                                                                                 PATCHER_checkcast_instanceof_flags,
3425                                                                                 (constant_classref *) iptr->target, 0);
3426
3427                                         if (opt_showdisassemble) {
3428                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3429                                         }
3430
3431                                         M_IMOV_IMM(0, REG_ITMP2);                 /* super->flags */
3432                                         M_IAND_IMM(ACC_INTERFACE, REG_ITMP2);
3433                                         M_BEQ(s2 + 5);
3434                                 }
3435
3436                                 /* interface checkcast code */
3437
3438                                 if (!super || (super->flags & ACC_INTERFACE)) {
3439                                         if (super) {
3440                                                 M_TEST(s1);
3441                                                 M_BEQ(s2);
3442                                         }
3443
3444                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3445
3446                                         if (!super) {
3447                                                 codegen_addpatchref(cd, cd->mcodeptr,
3448                                                                                         PATCHER_checkcast_instanceof_interface,
3449                                                                                         (constant_classref *) iptr->target, 0);
3450
3451                                                 if (opt_showdisassemble) {
3452                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3453                                                 }
3454                                         }
3455
3456                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3457                                                                                           OFFSET(vftbl_t, interfacetablelength),
3458                                                                                           REG_ITMP3);
3459                                         /* XXX TWISTI: should this be int arithmetic? */
3460                                         M_LSUB_IMM32(superindex, REG_ITMP3);
3461                                         M_TEST(REG_ITMP3);
3462                                         M_BLE(0);
3463                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3464                                         x86_64_mov_membase32_reg(cd, REG_ITMP2,
3465                                                                                          OFFSET(vftbl_t, interfacetable[0]) -
3466                                                                                          superindex * sizeof(methodptr*),
3467                                                                                          REG_ITMP3);
3468                                         M_TEST(REG_ITMP3);
3469                                         M_BEQ(0);
3470                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3471
3472                                         if (!super)
3473                                                 M_JMP_IMM(s3);
3474                                 }
3475
3476                                 /* class checkcast code */
3477
3478                                 if (!super || !(super->flags & ACC_INTERFACE)) {
3479                                         if (super) {
3480                                                 M_TEST(s1);
3481                                                 M_BEQ(s3);
3482                                         }
3483
3484                                         M_ALD(REG_ITMP2, s1, OFFSET(java_objectheader, vftbl));
3485
3486                                         if (!super) {
3487                                                 codegen_addpatchref(cd, cd->mcodeptr,
3488                                                                                         PATCHER_checkcast_class,
3489                                                                                         (constant_classref *) iptr->target,
3490                                                                                         0);
3491
3492                                                 if (opt_showdisassemble) {
3493                                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3494                                                 }
3495                                         }
3496
3497                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3498 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3499                                         codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3500 #endif
3501                                         x86_64_movl_membase32_reg(cd, REG_ITMP2,
3502                                                                                           OFFSET(vftbl_t, baseval),
3503                                                                                           REG_ITMP2);
3504                                         /*                                      if (s1 != REG_ITMP1) { */
3505                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3506                                         /*                                                                                              OFFSET(vftbl_t, baseval), */
3507                                         /*                                                                                              REG_ITMP1); */
3508                                         /*                                              x86_64_movl_membase_reg(cd, REG_ITMP3, */
3509                                         /*                                                                                              OFFSET(vftbl_t, diffval), */
3510                                         /*                                                                                              REG_ITMP3); */
3511                                         /*  #if defined(USE_THREADS) && defined(NATIVE_THREADS) */
3512                                         /*                                              codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase); */
3513                                         /*  #endif */
3514                                         /*                                              x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP1, REG_ITMP2); */
3515
3516                                         /*                                      } else { */
3517                                         x86_64_movl_membase32_reg(cd, REG_ITMP3,
3518                                                                                           OFFSET(vftbl_t, baseval),
3519                                                                                           REG_ITMP3);
3520                                         M_LSUB(REG_ITMP3, REG_ITMP2);
3521                                         M_MOV_IMM(supervftbl, REG_ITMP3);
3522                                         M_ILD(REG_ITMP3, REG_ITMP3, OFFSET(vftbl_t, diffval));
3523                                         /*                                      } */
3524 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3525                                         codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3526 #endif
3527                                         M_CMP(REG_ITMP3, REG_ITMP2);
3528                                         M_BA(0);         /* (u) REG_ITMP1 > (u) REG_ITMP2 -> jump */
3529                                         codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3530                                 }
3531                                 d = reg_of_var(rd, iptr->dst, REG_ITMP3);
3532
3533                         } else {
3534                                 /* array type cast-check */
3535
3536                                 var_to_reg_int(s1, src, REG_ITMP1);
3537                                 M_INTMOVE(s1, rd->argintregs[0]);
3538
3539                                 if (iptr->val.a == NULL) {
3540                                         codegen_addpatchref(cd, cd->mcodeptr,
3541                                                                                 PATCHER_builtin_arraycheckcast,
3542                                                                                 (constant_classref *) iptr->target, 0);
3543
3544                                         if (opt_showdisassemble) {
3545                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3546                                         }
3547                                 }
3548
3549                                 M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3550                                 M_MOV_IMM(BUILTIN_arraycheckcast, REG_ITMP1);
3551                                 M_CALL(REG_ITMP1);
3552                                 M_TEST(REG_RESULT);
3553                                 M_BEQ(0);
3554                                 codegen_add_classcastexception_ref(cd, cd->mcodeptr);
3555
3556                                 var_to_reg_int(s1, src, REG_ITMP1);
3557                                 d = reg_of_var(rd, iptr->dst, REG_ITMP1);
3558                         }
3559                         M_INTMOVE(s1, d);
3560                         store_reg_to_var_int(iptr->dst, d);
3561                         break;
3562
3563                 case ICMD_INSTANCEOF: /* ..., objectref ==> ..., intresult            */
3564
3565                                       /* op1:   0 == array, 1 == class                */
3566                                       /* val.a: (classinfo *) superclass              */
3567
3568                         /*  superclass is an interface:
3569                          *
3570                          *  return (sub != NULL) &&
3571                          *         (sub->vftbl->interfacetablelength > super->index) &&
3572                          *         (sub->vftbl->interfacetable[-super->index] != NULL);
3573                          *
3574                          *  superclass is a class:
3575                          *
3576                          *  return ((sub != NULL) && (0
3577                          *          <= (sub->vftbl->baseval - super->vftbl->baseval) <=
3578                          *          super->vftbl->diffvall));
3579                          */
3580
3581                         {
3582                         classinfo *super;
3583                         vftbl_t   *supervftbl;
3584                         s4         superindex;
3585
3586                         super = (classinfo *) iptr->val.a;
3587
3588                         if (!super) {
3589                                 superindex = 0;
3590                                 supervftbl = NULL;
3591
3592                         } else {
3593                                 superindex = super->index;
3594                                 supervftbl = super->vftbl;
3595                         }
3596
3597 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3598             codegen_threadcritrestart(cd, cd->mcodeptr - cd->mcodebase);
3599 #endif
3600
3601                         var_to_reg_int(s1, src, REG_ITMP1);
3602                         d = reg_of_var(rd, iptr->dst, REG_ITMP2);
3603                         if (s1 == d) {
3604                                 M_INTMOVE(s1, REG_ITMP1);
3605                                 s1 = REG_ITMP1;
3606                         }
3607
3608                         /* calculate interface instanceof code size */
3609
3610                         s2 = 3; /* mov_membase_reg */
3611                         CALCOFFSETBYTES(s2, s1, OFFSET(java_objectheader, vftbl));
3612                         s2 += 3 + 4 /* movl_membase32_reg */ + 3 + 4 /* sub_imm32 */ +
3613                                 3 /* test */ + 6 /* jcc */ + 3 + 4 /* mov_membase32_reg */ +
3614                                 3 /* test */ + 4 /* setcc */;
3615
3616                         if (!super)
3617                                 s2 += (opt_showdisassemble ? 5 : 0);
3618
3619                         /* calculate class instanceof code size */
3620
3621                         s3 = 3; /* mov_membase_reg */
3622                         CALCOFFSETBYTES(s3, s1, OFFSET(java_objectheader, vftbl));
3623                         s3 += 10; /* mov_imm_reg */
3624                         s3 += 2; /* movl_membase_reg - only if REG_ITMP1 == RAX */
3625                         CALCOFFSETBYTES(s3, REG_ITMP1, OFFSET(vftbl_t, baseval));
3626                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3627                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, baseval));
3628                         s3 += 3; /* movl_membase_reg - only if REG_ITMP2 == R10 */
3629                         CALCOFFSETBYTES(s3, REG_ITMP2, OFFSET(vftbl_t, diffval));
3630                         s3 += 3 /* sub */ + 3 /* xor */ + 3 /* cmp */ + 4 /* setcc */;
3631
3632                         if (!super)
3633                                 s3 += (opt_showdisassemble ? 5 : 0);
3634
3635                         x86_64_alu_reg_reg(cd, X86_64_XOR, d, d);
3636
3637                         /* if class is not resolved, check which code to call */
3638
3639                         if (!super) {
3640                                 x86_64_test_reg_reg(cd, s1, s1);
3641                                 x86_64_jcc(cd, X86_64_CC_Z, (6 + (opt_showdisassemble ? 5 : 0) +
3642                                                                                          7 + 6 + s2 + 5 + s3));
3643
3644                                 codegen_addpatchref(cd, cd->mcodeptr,
3645                                                                         PATCHER_checkcast_instanceof_flags,
3646                                                                         (constant_classref *) iptr->target, 0);
3647
3648                                 if (opt_showdisassemble) {
3649                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3650                                 }
3651
3652                                 x86_64_movl_imm_reg(cd, 0, REG_ITMP3); /* super->flags */
3653                                 x86_64_alul_imm_reg(cd, X86_64_AND, ACC_INTERFACE, REG_ITMP3);
3654                                 x86_64_jcc(cd, X86_64_CC_Z, s2 + 5);
3655                         }
3656
3657                         /* interface instanceof code */
3658
3659                         if (!super || (super->flags & ACC_INTERFACE)) {
3660                                 if (super) {
3661                                         x86_64_test_reg_reg(cd, s1, s1);
3662                                         x86_64_jcc(cd, X86_64_CC_Z, s2);
3663                                 }
3664
3665                                 x86_64_mov_membase_reg(cd, s1,
3666                                                                            OFFSET(java_objectheader, vftbl),
3667                                                                            REG_ITMP1);
3668                                 if (!super) {
3669                                         codegen_addpatchref(cd, cd->mcodeptr,
3670                                                                                 PATCHER_checkcast_instanceof_interface,
3671                                                                                 (constant_classref *) iptr->target, 0);
3672
3673                                         if (opt_showdisassemble) {
3674                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3675                                         }
3676                                 }
3677
3678                                 x86_64_movl_membase32_reg(cd, REG_ITMP1,
3679                                                                                   OFFSET(vftbl_t, interfacetablelength),
3680                                                                                   REG_ITMP3);
3681                                 x86_64_alu_imm32_reg(cd, X86_64_SUB, superindex, REG_ITMP3);
3682                                 x86_64_test_reg_reg(cd, REG_ITMP3, REG_ITMP3);
3683
3684                                 a = 3 + 4 /* mov_membase32_reg */ + 3 /* test */ + 4 /* setcc */;
3685
3686                                 x86_64_jcc(cd, X86_64_CC_LE, a);
3687                                 x86_64_mov_membase32_reg(cd, REG_ITMP1,
3688                                                                                  OFFSET(vftbl_t, interfacetable[0]) -
3689                                                                                  superindex * sizeof(methodptr*),
3690                                                                                  REG_ITMP1);
3691                                 x86_64_test_reg_reg(cd, REG_ITMP1, REG_ITMP1);
3692                                 x86_64_setcc_reg(cd, X86_64_CC_NE, d);
3693
3694                                 if (!super)
3695                                         x86_64_jmp_imm(cd, s3);
3696                         }
3697
3698                         /* class instanceof code */
3699
3700                         if (!super || !(super->flags & ACC_INTERFACE)) {
3701                                 if (super) {
3702                                         x86_64_test_reg_reg(cd, s1, s1);
3703                                         x86_64_jcc(cd, X86_64_CC_E, s3);
3704                                 }
3705
3706                                 x86_64_mov_membase_reg(cd, s1,
3707                                                                            OFFSET(java_objectheader, vftbl),
3708                                                                            REG_ITMP1);
3709
3710                                 if (!super) {
3711                                         codegen_addpatchref(cd, cd->mcodeptr,
3712                                                                                 PATCHER_instanceof_class,
3713                                                                                 (constant_classref *) iptr->target, 0);
3714
3715                                         if (opt_showdisassemble) {
3716                                                 M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3717                                         }
3718                                 }
3719
3720                                 x86_64_mov_imm_reg(cd, (ptrint) supervftbl, REG_ITMP2);
3721 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3722                                 codegen_threadcritstart(cd, cd->mcodeptr - cd->mcodebase);
3723 #endif
3724                                 x86_64_movl_membase_reg(cd, REG_ITMP1,
3725                                                                                 OFFSET(vftbl_t, baseval),
3726                                                                                 REG_ITMP1);
3727                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3728                                                                                 OFFSET(vftbl_t, diffval),
3729                                                                                 REG_ITMP3);
3730                                 x86_64_movl_membase_reg(cd, REG_ITMP2,
3731                                                                                 OFFSET(vftbl_t, baseval),
3732                                                                                 REG_ITMP2);
3733 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3734                                 codegen_threadcritstop(cd, cd->mcodeptr - cd->mcodebase);
3735 #endif
3736                                 x86_64_alu_reg_reg(cd, X86_64_SUB, REG_ITMP2, REG_ITMP1);
3737                                 x86_64_alu_reg_reg(cd, X86_64_XOR, d, d); /* may be REG_ITMP2 */
3738                                 x86_64_alu_reg_reg(cd, X86_64_CMP, REG_ITMP3, REG_ITMP1);
3739                                 x86_64_setcc_reg(cd, X86_64_CC_BE, d);
3740                         }
3741                         store_reg_to_var_int(iptr->dst, d);
3742                         }
3743                         break;
3744
3745                 case ICMD_MULTIANEWARRAY:/* ..., cnt1, [cnt2, ...] ==> ..., arrayref  */
3746                                          /* op1 = dimension, val.a = class            */
3747
3748                         /* check for negative sizes and copy sizes to stack if necessary  */
3749
3750                         MCODECHECK((10 * 4 * iptr->op1) + 5 + 10 * 8);
3751
3752                         for (s1 = iptr->op1; --s1 >= 0; src = src->prev) {
3753                                 /* copy SAVEDVAR sizes to stack */
3754
3755                                 if (src->varkind != ARGVAR) {
3756                                         var_to_reg_int(s2, src, REG_ITMP1);
3757                                         M_LST(s2, REG_SP, s1 * 8);
3758                                 }
3759                         }
3760
3761                         /* is a patcher function set? */
3762
3763                         if (iptr->val.a == NULL) {
3764                                 codegen_addpatchref(cd, cd->mcodeptr,
3765                                                                         PATCHER_builtin_multianewarray,
3766                                                                         (constant_classref *) iptr->target, 0);
3767
3768                                 if (opt_showdisassemble) {
3769                                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
3770                                 }
3771
3772                                 a = 0;
3773
3774                         } else {
3775                                 a = (ptrint) iptr->val.a;
3776                         }
3777
3778                         /* a0 = dimension count */
3779
3780                         M_MOV_IMM(iptr->op1, rd->argintregs[0]);
3781
3782                         /* a1 = arrayvftbl */
3783
3784                         M_MOV_IMM(iptr->val.a, rd->argintregs[1]);
3785
3786                         /* a2 = pointer to dimensions = stack pointer */
3787
3788                         M_MOV(REG_SP, rd->argintregs[2]);
3789
3790                         M_MOV_IMM(BUILTIN_multianewarray, REG_ITMP1);
3791                         M_CALL(REG_ITMP1);
3792
3793                         /* check for exception before result assignment */
3794
3795                         M_TEST(REG_RESULT);
3796                         M_BEQ(0);
3797                         codegen_add_fillinstacktrace_ref(cd, cd->mcodeptr);
3798
3799                         s1 = reg_of_var(rd, iptr->dst, REG_RESULT);
3800                         M_INTMOVE(REG_RESULT, s1);
3801                         store_reg_to_var_int(iptr->dst, s1);
3802                         break;
3803
3804                 default:
3805                         *exceptionptr = new_internalerror("Unknown ICMD %d", iptr->opc);
3806                         return false;
3807         } /* switch */
3808
3809         } /* for instruction */
3810
3811         /* copy values to interface registers */
3812
3813         src = bptr->outstack;
3814         len = bptr->outdepth;
3815         MCODECHECK(512);
3816 #if defined(ENABLE_LSRA)
3817         if (!opt_lsra)
3818 #endif
3819         while (src) {
3820                 len--;
3821                 if ((src->varkind != STACKVAR)) {
3822                         s2 = src->type;
3823                         if (IS_FLT_DBL_TYPE(s2)) {
3824                                 var_to_reg_flt(s1, src, REG_FTMP1);
3825                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3826                                         M_FLTMOVE(s1, rd->interfaces[len][s2].regoff);
3827
3828                                 } else {
3829                                         x86_64_movq_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3830                                 }
3831
3832                         } else {
3833                                 var_to_reg_int(s1, src, REG_ITMP1);
3834                                 if (!(rd->interfaces[len][s2].flags & INMEMORY)) {
3835                                         M_INTMOVE(s1, rd->interfaces[len][s2].regoff);
3836
3837                                 } else {
3838                                         x86_64_mov_reg_membase(cd, s1, REG_SP, rd->interfaces[len][s2].regoff * 8);
3839                                 }
3840                         }
3841                 }
3842                 src = src->prev;
3843         }
3844
3845         /* At the end of a basic block we may have to append some nops,
3846            because the patcher stub calling code might be longer than the
3847            actual instruction. So codepatching does not change the
3848            following block unintentionally. */
3849
3850         if (cd->mcodeptr < cd->lastmcodeptr) {
3851                 while (cd->mcodeptr < cd->lastmcodeptr) {
3852                         M_NOP;
3853                 }
3854         }
3855
3856         } /* if (bptr -> flags >= BBREACHED) */
3857         } /* for basic block */
3858
3859         dseg_createlinenumbertable(cd);
3860
3861
3862         /* generate exception and patcher stubs */
3863
3864         {
3865                 exceptionref *eref;
3866                 patchref     *pref;
3867                 ptrint        mcode;
3868                 u1           *savedmcodeptr;
3869                 u1           *tmpmcodeptr;
3870
3871                 savedmcodeptr = NULL;
3872
3873                 /* generate exception stubs */
3874
3875                 for (eref = cd->exceptionrefs; eref != NULL; eref = eref->next) {
3876                         gen_resolvebranch(cd->mcodebase + eref->branchpos,
3877                                                           eref->branchpos,
3878                                                           cd->mcodeptr - cd->mcodebase);
3879
3880                         MCODECHECK(512);
3881
3882                         /* Check if the exception is an
3883                            ArrayIndexOutOfBoundsException.  If so, move index register
3884                            into REG_ITMP1. */
3885
3886                         if (eref->reg != -1)
3887                                 M_MOV(eref->reg, REG_ITMP1);
3888
3889                         /* calcuate exception address */
3890
3891                         M_MOV_IMM(0, REG_ITMP2_XPC);
3892                         dseg_adddata(cd, cd->mcodeptr);
3893                         M_AADD_IMM32(eref->branchpos - 6, REG_ITMP2_XPC);
3894
3895                         /* move function to call into REG_ITMP3 */
3896
3897                         M_MOV_IMM(eref->function, REG_ITMP3);
3898
3899                         if (savedmcodeptr != NULL) {
3900                                 M_JMP_IMM(savedmcodeptr - cd->mcodeptr - 5);
3901
3902                         } else {
3903                                 savedmcodeptr = cd->mcodeptr;
3904
3905                                 x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[0]);
3906                                 M_MOV(REG_SP, rd->argintregs[1]);
3907                                 M_ALD(rd->argintregs[2], REG_SP, parentargs_base * 8);
3908                                 M_MOV(REG_ITMP2_XPC, rd->argintregs[3]);
3909                                 M_MOV(REG_ITMP1, rd->argintregs[4]);            /* for AIOOBE */
3910
3911                                 M_ASUB_IMM(2 * 8, REG_SP);
3912                                 M_AST(REG_ITMP2_XPC, REG_SP, 0 * 8);
3913
3914                                 M_CALL(REG_ITMP3);
3915
3916                                 M_ALD(REG_ITMP2_XPC, REG_SP, 0 * 8);
3917                                 M_AADD_IMM(2 * 8, REG_SP);
3918
3919                                 M_MOV_IMM(asm_handle_exception, REG_ITMP3);
3920                                 M_JMP(REG_ITMP3);
3921                         }
3922                 }
3923
3924
3925                 /* generate code patching stub call code */
3926
3927                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
3928                         /* check size of code segment */
3929
3930                         MCODECHECK(512);
3931
3932                         /* Get machine code which is patched back in later. A
3933                            `call rel32' is 5 bytes long (but read 8 bytes). */
3934
3935                         savedmcodeptr = cd->mcodebase + pref->branchpos;
3936                         mcode = *((ptrint *) savedmcodeptr);
3937
3938                         /* patch in `call rel32' to call the following code */
3939
3940                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
3941                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
3942
3943                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
3944
3945                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
3946
3947                         /* move pointer to java_objectheader onto stack */
3948
3949 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
3950                         /* create a virtual java_objectheader */
3951
3952                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
3953                         a = dseg_addaddress(cd, NULL);                      /* vftbl      */
3954
3955                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + a, REG_ITMP3);
3956                         M_PUSH(REG_ITMP3);
3957 #else
3958                         M_PUSH_IMM(0);
3959 #endif
3960
3961                         /* move machine code bytes and classinfo pointer into registers */
3962
3963                         M_MOV_IMM(mcode, REG_ITMP3);
3964                         M_PUSH(REG_ITMP3);
3965                         M_MOV_IMM(pref->ref, REG_ITMP3);
3966                         M_PUSH(REG_ITMP3);
3967                         M_MOV_IMM(pref->disp, REG_ITMP3);
3968                         M_PUSH(REG_ITMP3);
3969
3970                         M_MOV_IMM(pref->patcher, REG_ITMP3);
3971                         M_PUSH(REG_ITMP3);
3972
3973                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
3974                         M_JMP(REG_ITMP3);
3975                 }
3976         }
3977
3978         /* generate replacement-out stubs */
3979
3980         {
3981                 int i;
3982
3983                 replacementpoint = cd->code->rplpoints;
3984                 for (i=0; i<cd->code->rplpointcount; ++i, ++replacementpoint) {
3985                         /* check code segment size */
3986
3987                         MCODECHECK(512);
3988
3989                         /* note start of stub code */
3990
3991                         replacementpoint->outcode = (u1*) (ptrint)(cd->mcodeptr - cd->mcodebase);
3992
3993                         /* make machine code for patching */
3994
3995                         disp = (ptrint)(replacementpoint->outcode - replacementpoint->pc) - 5;
3996                         replacementpoint->mcode = 0xe9 | ((u8)disp << 8);
3997
3998                         /* push address of `rplpoint` struct */
3999
4000                         M_MOV_IMM(replacementpoint, REG_ITMP3);
4001                         M_PUSH(REG_ITMP3);
4002
4003                         /* jump to replacement function */
4004
4005                         M_MOV_IMM(asm_replacement_out, REG_ITMP3);
4006                         M_JMP(REG_ITMP3);
4007                 }
4008         }
4009
4010         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4011
4012         /* everything's ok */
4013
4014         return true;
4015 }
4016
4017
4018 /* createcompilerstub **********************************************************
4019
4020    Creates a stub routine which calls the compiler.
4021
4022 *******************************************************************************/
4023
4024 #define COMPILERSTUB_DATASIZE    2 * SIZEOF_VOID_P
4025 #define COMPILERSTUB_CODESIZE    7 + 7 + 3
4026
4027 #define COMPILERSTUB_SIZE        COMPILERSTUB_DATASIZE + COMPILERSTUB_CODESIZE
4028
4029
4030 u1 *createcompilerstub(methodinfo *m)
4031 {
4032         u1          *s;                     /* memory to hold the stub            */
4033         ptrint      *d;
4034         codegendata *cd;
4035         s4           dumpsize;
4036
4037         s = CNEW(u1, COMPILERSTUB_SIZE);
4038
4039         /* set data pointer and code pointer */
4040
4041         d = (ptrint *) s;
4042         s = s + COMPILERSTUB_DATASIZE;
4043
4044         /* mark start of dump memory area */
4045
4046         dumpsize = dump_size();
4047
4048         cd = DNEW(codegendata);
4049         cd->mcodeptr = s;
4050
4051         /* Store the methodinfo* in the same place as in the methodheader
4052            for compiled methods. */
4053
4054         d[0] = (ptrint) asm_call_jit_compiler;
4055         d[1] = (ptrint) m;
4056
4057         /* code for the stub */
4058
4059         M_ALD(REG_ITMP1, RIP, -(7 * 1 + 1 * SIZEOF_VOID_P));       /* methodinfo  */
4060         M_ALD(REG_ITMP3, RIP, -(7 * 2 + 2 * SIZEOF_VOID_P));  /* compiler pointer */
4061         M_JMP(REG_ITMP3);
4062
4063 #if defined(ENABLE_STATISTICS)
4064         if (opt_stat)
4065                 count_cstub_len += COMPILERSTUB_SIZE;
4066 #endif
4067
4068         /* release dump area */
4069
4070         dump_release(dumpsize);
4071
4072         return s;
4073 }
4074
4075
4076 /* createnativestub ************************************************************
4077
4078    Creates a stub routine which calls a native method.
4079
4080 *******************************************************************************/
4081
4082 u1 *createnativestub(functionptr f, methodinfo *m, codegendata *cd,
4083                                          registerdata *rd, methoddesc *nmd)
4084 {
4085         methoddesc *md;
4086         s4          stackframesize;         /* size of stackframe if needed       */
4087         s4          nativeparams;
4088         s4          i, j;                   /* count variables                    */
4089         s4          t;
4090         s4          s1, s2;
4091
4092         /* initialize variables */
4093
4094         md = m->parseddesc;
4095         nativeparams = (m->flags & ACC_STATIC) ? 2 : 1;
4096
4097         /* calculate stack frame size */
4098
4099         stackframesize =
4100                 sizeof(stackframeinfo) / SIZEOF_VOID_P +
4101                 sizeof(localref_table) / SIZEOF_VOID_P +
4102                 INT_ARG_CNT + FLT_ARG_CNT + 1 +         /* + 1 for function address   */
4103                 nmd->memuse;
4104
4105         if (!(stackframesize & 0x1))                /* keep stack 16-byte aligned */
4106                 stackframesize++;
4107
4108         /* create method header */
4109
4110         (void) dseg_addaddress(cd, m);                          /* MethodPointer  */
4111         (void) dseg_adds4(cd, stackframesize * 8);              /* FrameSize      */
4112         (void) dseg_adds4(cd, 0);                               /* IsSync         */
4113         (void) dseg_adds4(cd, 0);                               /* IsLeaf         */
4114         (void) dseg_adds4(cd, 0);                               /* IntSave        */
4115         (void) dseg_adds4(cd, 0);                               /* FltSave        */
4116         (void) dseg_addlinenumbertablesize(cd);
4117         (void) dseg_adds4(cd, 0);                               /* ExTableSize    */
4118
4119         /* initialize mcode variables */
4120
4121         cd->mcodeptr = (u1 *) cd->mcodebase;
4122         cd->mcodeend = (s4 *) (cd->mcodebase + cd->mcodesize);
4123
4124         /* generate native method profiling code */
4125
4126         if (opt_prof) {
4127                 /* count frequency */
4128
4129                 M_MOV_IMM(m, REG_ITMP2);
4130                 M_IINC_MEMBASE(REG_ITMP2, OFFSET(methodinfo, frequency));
4131         }
4132
4133         /* generate stub code */
4134
4135         M_ASUB_IMM(stackframesize * 8, REG_SP);
4136
4137 #if !defined(NDEBUG)
4138         /* generate call trace */
4139
4140         if (opt_verbosecall) {
4141                 /* save integer and float argument registers */
4142
4143                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4144                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4145                                 M_LST(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4146
4147                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4148                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4149                                 M_DST(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4150
4151                 /* show integer hex code for float arguments */
4152
4153                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++) {
4154                         /* if the paramtype is a float, we have to right shift all
4155                            following integer registers */
4156
4157                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type)) {
4158                                 for (s1 = INT_ARG_CNT - 2; s1 >= i; s1--)
4159                                         M_MOV(rd->argintregs[s1], rd->argintregs[s1 + 1]);
4160
4161                                 x86_64_movd_freg_reg(cd, rd->argfltregs[j], rd->argintregs[i]);
4162                                 j++;
4163                         }
4164                 }
4165
4166                 M_MOV_IMM(m, REG_ITMP1);
4167                 M_AST(REG_ITMP1, REG_SP, 0 * 8);
4168                 M_MOV_IMM(builtin_trace_args, REG_ITMP1);
4169                 M_CALL(REG_ITMP1);
4170
4171                 /* restore integer and float argument registers */
4172
4173                 for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4174                         if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4175                                 M_LLD(rd->argintregs[j++], REG_SP, (1 + i) * 8);
4176
4177                 for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4178                         if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4179                                 M_DLD(rd->argfltregs[j++], REG_SP, (1 + INT_ARG_CNT + i) * 8);
4180         }
4181 #endif /* !defined(NDEBUG) */
4182
4183         /* get function address (this must happen before the stackframeinfo) */
4184
4185 #if !defined(WITH_STATIC_CLASSPATH)
4186         if (f == NULL) {
4187                 codegen_addpatchref(cd, cd->mcodeptr, PATCHER_resolve_native, m, 0);
4188
4189                 if (opt_showdisassemble) {
4190                         M_NOP; M_NOP; M_NOP; M_NOP; M_NOP;
4191                 }
4192         }
4193 #endif
4194
4195         M_MOV_IMM(f, REG_ITMP3);
4196
4197
4198         /* save integer and float argument registers */
4199
4200         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4201                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4202                         M_LST(rd->argintregs[j++], REG_SP, i * 8);
4203
4204         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4205                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4206                         M_DST(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4207
4208         M_AST(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4209
4210         /* create dynamic stack info */
4211
4212         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4213         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase), rd->argintregs[1]);
4214         M_ALEA(REG_SP, stackframesize * 8 + SIZEOF_VOID_P, rd->argintregs[2]);
4215         M_ALD(rd->argintregs[3], REG_SP, stackframesize * 8);
4216         M_MOV_IMM(codegen_start_native_call, REG_ITMP1);
4217         M_CALL(REG_ITMP1);
4218
4219         /* restore integer and float argument registers */
4220
4221         for (i = 0, j = 0; i < md->paramcount && j < INT_ARG_CNT; i++)
4222                 if (IS_INT_LNG_TYPE(md->paramtypes[i].type))
4223                         M_LLD(rd->argintregs[j++], REG_SP, i * 8);
4224
4225         for (i = 0, j = 0; i < md->paramcount && j < FLT_ARG_CNT; i++)
4226                 if (IS_FLT_DBL_TYPE(md->paramtypes[i].type))
4227                         M_DLD(rd->argfltregs[j++], REG_SP, (INT_ARG_CNT + i) * 8);
4228
4229         M_ALD(REG_ITMP3, REG_SP, (INT_ARG_CNT + FLT_ARG_CNT) * 8);
4230
4231
4232         /* copy or spill arguments to new locations */
4233
4234         for (i = md->paramcount - 1, j = i + nativeparams; i >= 0; i--, j--) {
4235                 t = md->paramtypes[i].type;
4236
4237                 if (IS_INT_LNG_TYPE(t)) {
4238                         if (!md->params[i].inmemory) {
4239                                 s1 = rd->argintregs[md->params[i].regoff];
4240
4241                                 if (!nmd->params[j].inmemory) {
4242                                         s2 = rd->argintregs[nmd->params[j].regoff];
4243                                         M_INTMOVE(s1, s2);
4244
4245                                 } else {
4246                                         s2 = nmd->params[j].regoff;
4247                                         M_LST(s1, REG_SP, s2 * 8);
4248                                 }
4249
4250                         } else {
4251                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4252                                 s2 = nmd->params[j].regoff;
4253                                 M_LLD(REG_ITMP1, REG_SP, s1 * 8);
4254                                 M_LST(REG_ITMP1, REG_SP, s2 * 8);
4255                         }
4256
4257                 } else {
4258                         /* We only copy spilled float arguments, as the float argument    */
4259                         /* registers keep unchanged.                                      */
4260
4261                         if (md->params[i].inmemory) {
4262                                 s1 = md->params[i].regoff + stackframesize + 1;   /* + 1 (RA) */
4263                                 s2 = nmd->params[j].regoff;
4264                                 M_DLD(REG_FTMP1, REG_SP, s1 * 8);
4265                                 M_DST(REG_FTMP1, REG_SP, s2 * 8);
4266                         }
4267                 }
4268         }
4269
4270         /* put class into second argument register */
4271
4272         if (m->flags & ACC_STATIC)
4273                 M_MOV_IMM(m->class, rd->argintregs[1]);
4274
4275         /* put env into first argument register */
4276
4277         M_MOV_IMM(_Jv_env, rd->argintregs[0]);
4278
4279         /* do the native function call */
4280
4281         M_CALL(REG_ITMP3);
4282
4283         /* save return value */
4284
4285         if (md->returntype.type != TYPE_VOID) {
4286                 if (IS_INT_LNG_TYPE(md->returntype.type))
4287                         M_LST(REG_RESULT, REG_SP, 0 * 8);
4288                 else
4289                         M_DST(REG_FRESULT, REG_SP, 0 * 8);
4290         }
4291
4292         /* remove native stackframe info */
4293
4294         M_ALEA(REG_SP, stackframesize * 8, rd->argintregs[0]);
4295         M_MOV_IMM(codegen_finish_native_call, REG_ITMP1);
4296         M_CALL(REG_ITMP1);
4297
4298 #if !defined(NDEBUG)
4299         /* generate call trace */
4300
4301         if (opt_verbosecall) {
4302                 /* just restore the value we need, don't care about the other */
4303
4304                 if (md->returntype.type != TYPE_VOID) {
4305                         if (IS_INT_LNG_TYPE(md->returntype.type))
4306                                 M_LLD(REG_RESULT, REG_SP, 0 * 8);
4307                         else
4308                                 M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4309                 }
4310
4311                 M_MOV_IMM(m, rd->argintregs[0]);
4312                 M_MOV(REG_RESULT, rd->argintregs[1]);
4313                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[0]);
4314                 M_FLTMOVE(REG_FRESULT, rd->argfltregs[1]);
4315
4316                 M_MOV_IMM(builtin_displaymethodstop, REG_ITMP1);
4317                 M_CALL(REG_ITMP1);
4318         }
4319 #endif /* !defined(NDEBUG) */
4320
4321         /* check for exception */
4322
4323 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4324         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4325         M_CALL(REG_ITMP3);
4326 #else
4327         M_MOV_IMM(&_no_threads_exceptionptr, REG_RESULT);
4328 #endif
4329         M_ALD(REG_ITMP2, REG_RESULT, 0);
4330
4331         /* restore return value */
4332
4333         if (md->returntype.type != TYPE_VOID) {
4334                 if (IS_INT_LNG_TYPE(md->returntype.type))
4335                         M_LLD(REG_RESULT, REG_SP, 0 * 8);
4336                 else
4337                         M_DLD(REG_FRESULT, REG_SP, 0 * 8);
4338         }
4339
4340         /* test for exception */
4341
4342         M_TEST(REG_ITMP2);
4343         M_BNE(7 + 1);
4344
4345         /* remove stackframe */
4346
4347         M_AADD_IMM(stackframesize * 8, REG_SP);
4348         M_RET;
4349
4350
4351         /* handle exception */
4352
4353 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4354         M_LST(REG_ITMP2, REG_SP, 0 * 8);
4355         M_MOV_IMM(builtin_get_exceptionptrptr, REG_ITMP3);
4356         M_CALL(REG_ITMP3);
4357         M_AST_IMM32(0, REG_RESULT, 0);                 /* clear exception pointer */
4358         M_LLD(REG_ITMP1_XPTR, REG_SP, 0 * 8);
4359 #else
4360         M_MOV(REG_ITMP3, REG_ITMP1_XPTR);
4361         M_MOV_IMM(&_no_threads_exceptionptr, REG_ITMP3);
4362         M_AST_IMM32(0, REG_ITMP3, 0);                  /* clear exception pointer */
4363 #endif
4364
4365         /* remove stackframe */
4366
4367         M_AADD_IMM(stackframesize * 8, REG_SP);
4368
4369         M_LLD(REG_ITMP2_XPC, REG_SP, 0 * 8);     /* get return address from stack */
4370         M_ASUB_IMM(3, REG_ITMP2_XPC);                                    /* callq */
4371
4372         M_MOV_IMM(asm_handle_nat_exception, REG_ITMP3);
4373         M_JMP(REG_ITMP3);
4374
4375
4376         /* process patcher calls **************************************************/
4377
4378         {
4379                 patchref *pref;
4380                 ptrint    mcode;
4381                 u1       *savedmcodeptr;
4382                 u1       *tmpmcodeptr;
4383 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4384                 s4        disp;
4385 #endif
4386
4387                 for (pref = cd->patchrefs; pref != NULL; pref = pref->next) {
4388                         /* Get machine code which is patched back in later. A
4389                            `call rel32' is 5 bytes long (but read 8 bytes). */
4390
4391                         savedmcodeptr = cd->mcodebase + pref->branchpos;
4392                         mcode = *((ptrint *) savedmcodeptr);
4393
4394                         /* patch in `call rel32' to call the following code               */
4395
4396                         tmpmcodeptr  = cd->mcodeptr;    /* save current mcodeptr          */
4397                         cd->mcodeptr = savedmcodeptr;   /* set mcodeptr to patch position */
4398
4399                         M_CALL_IMM(tmpmcodeptr - (savedmcodeptr + PATCHER_CALL_SIZE));
4400
4401                         cd->mcodeptr = tmpmcodeptr;     /* restore the current mcodeptr   */
4402
4403                         /* move pointer to java_objectheader onto stack */
4404
4405 #if defined(USE_THREADS) && defined(NATIVE_THREADS)
4406                         /* create a virtual java_objectheader */
4407
4408                         (void) dseg_addaddress(cd, get_dummyLR());          /* monitorPtr */
4409                         disp = dseg_addaddress(cd, NULL);                   /* vftbl      */
4410
4411                         x86_64_lea_membase_reg(cd, RIP, -(((ptrint) cd->mcodeptr + 7) - (ptrint) cd->mcodebase) + disp, REG_ITMP3);
4412                         M_PUSH(REG_ITMP3);
4413 #else
4414                         M_PUSH_IMM(0);
4415 #endif
4416
4417                         /* move machine code bytes and classinfo pointer into registers */
4418
4419                         M_MOV_IMM(mcode, REG_ITMP3);
4420                         M_PUSH(REG_ITMP3);
4421                         M_MOV_IMM(pref->ref, REG_ITMP3);
4422                         M_PUSH(REG_ITMP3);
4423                         M_MOV_IMM(pref->disp, REG_ITMP3);
4424                         M_PUSH(REG_ITMP3);
4425
4426                         M_MOV_IMM(pref->patcher, REG_ITMP3);
4427                         M_PUSH(REG_ITMP3);
4428
4429                         M_MOV_IMM(asm_wrapper_patcher, REG_ITMP3);
4430                         M_JMP(REG_ITMP3);
4431                 }
4432         }
4433
4434         codegen_finish(m, cd, (s4) ((u1 *) cd->mcodeptr - cd->mcodebase));
4435
4436         return cd->code->entrypoint;
4437 }
4438
4439
4440 /*
4441  * These are local overrides for various environment variables in Emacs.
4442  * Please do not remove this and leave it at the end of the file, where
4443  * Emacs will automagically detect them.
4444  * ---------------------------------------------------------------------
4445  * Local variables:
4446  * mode: c
4447  * indent-tabs-mode: t
4448  * c-basic-offset: 4
4449  * tab-width: 4
4450  * End:
4451  * vim:noexpandtab:sw=4:ts=4:
4452  */